From 8359dd864ac5e731746af7663e93468507dcdc28 Mon Sep 17 00:00:00 2001 From: zhang2014 Date: Sat, 28 Jun 2025 11:47:34 +0800 Subject: [PATCH 1/8] refactor(query): use trait to refactor physical plan --- src/query/sql/src/executor/mod.rs | 2 + src/query/sql/src/executor/physical_plan.rs | 812 ++++++++++-------- .../physical_add_stream_column.rs | 6 +- .../physical_aggregate_expand.rs | 50 +- .../physical_aggregate_final.rs | 40 +- .../physical_aggregate_partial.rs | 42 +- .../physical_plans/physical_async_func.rs | 15 +- .../physical_plans/physical_broadcast.rs | 8 +- .../physical_plans/physical_cache_scan.rs | 6 +- .../physical_column_mutation.rs | 6 +- .../physical_plans/physical_commit_sink.rs | 6 +- .../physical_plans/physical_compact_source.rs | 6 +- .../physical_constant_table_scan.rs | 6 +- .../physical_copy_into_location.rs | 6 +- .../physical_copy_into_table.rs | 5 + .../physical_distributed_insert_select.rs | 6 +- .../physical_plans/physical_eval_scalar.rs | 46 +- .../physical_plans/physical_exchange.rs | 6 +- .../physical_plans/physical_exchange_sink.rs | 6 +- .../physical_exchange_source.rs | 5 + .../physical_expression_scan.rs | 6 +- .../physical_plans/physical_filter.rs | 34 +- .../physical_plans/physical_hash_join.rs | 53 +- .../executor/physical_plans/physical_limit.rs | 23 +- .../physical_multi_table_insert.rs | 22 +- .../physical_plans/physical_mutation.rs | 6 +- .../physical_mutation_into_organize.rs | 5 + .../physical_mutation_into_split.rs | 6 +- .../physical_mutation_manipulate.rs | 5 + .../physical_mutation_source.rs | 6 +- .../physical_plans/physical_project_set.rs | 40 +- .../physical_plans/physical_r_cte_scan.rs | 6 +- .../physical_plans/physical_range_join.rs | 6 +- .../physical_plans/physical_recluster.rs | 10 +- .../physical_replace_async_source.rs | 6 +- .../physical_replace_deduplicate.rs | 6 +- .../physical_plans/physical_replace_into.rs | 6 +- .../physical_plans/physical_row_fetch.rs | 29 +- .../executor/physical_plans/physical_sort.rs | 61 +- .../physical_plans/physical_table_scan.rs | 73 +- .../executor/physical_plans/physical_udf.rs | 6 +- .../physical_plans/physical_union_all.rs | 6 +- .../physical_plans/physical_window.rs | 34 +- .../physical_window_partition.rs | 23 +- 44 files changed, 1127 insertions(+), 436 deletions(-) diff --git a/src/query/sql/src/executor/mod.rs b/src/query/sql/src/executor/mod.rs index 3e40852552cac..d9aa9b341a1d7 100644 --- a/src/query/sql/src/executor/mod.rs +++ b/src/query/sql/src/executor/mod.rs @@ -24,6 +24,8 @@ pub mod table_read_plan; pub use format::format_partial_tree; pub use physical_plan::PhysicalPlan; +pub use physical_plan::IPhysicalPlan; +pub use physical_plan::PhysicalPlanMeta; pub use physical_plan_builder::MutationBuildInfo; pub use physical_plan_builder::PhysicalPlanBuilder; pub use physical_plan_visitor::PhysicalPlanReplacer; diff --git a/src/query/sql/src/executor/physical_plan.rs b/src/query/sql/src/executor/physical_plan.rs index 33f29fabf5cd1..8f8fd4c30f335 100644 --- a/src/query/sql/src/executor/physical_plan.rs +++ b/src/query/sql/src/executor/physical_plan.rs @@ -13,7 +13,7 @@ // limitations under the License. use std::collections::HashMap; - +use std::fmt::Debug; use databend_common_catalog::plan::DataSourceInfo; use databend_common_catalog::plan::DataSourcePlan; use databend_common_catalog::plan::PartStatistics; @@ -79,6 +79,117 @@ use crate::executor::physical_plans::UnionAll; use crate::executor::physical_plans::Window; use crate::executor::physical_plans::WindowPartition; +#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] +pub struct PhysicalPlanMeta { + plan_id: u32, + name: String, + // for profile + desc: String, + // for profile + labels: HashMap>, +} + +impl PhysicalPlanMeta { + pub fn new(name: String) -> PhysicalPlanMeta { + PhysicalPlanMeta { + plan_id: 0, + name, + desc: String::new(), + labels: HashMap::new(), + } + } + + pub fn with_desc(name: String, desc: String, labels: HashMap>) -> PhysicalPlanMeta { + PhysicalPlanMeta { + plan_id: 0, + name, + desc, + labels, + } + } +} + +pub trait IPhysicalPlan: serde::Serialize + serde::Deserialize + Clone + Debug { + fn get_meta(&self) -> &PhysicalPlanMeta; + + fn get_meta_mut(&mut self) -> &mut PhysicalPlanMeta; + + // For methods with default implementations, the default implementation is usually sufficient. + + fn get_id(&self) -> u32 { + self.get_meta().plan_id + } + + fn get_name(&self) -> String { + self.get_meta().name.clone() + } + + #[recursive::recursive] + fn adjust_plan_id(&mut self, next_id: &mut u32) { + self.get_meta_mut().plan_id = *next_id; + *next_id += 1; + + for child in self.children_mut() { + child.adjust_plan_id(next_id); + } + } + + fn output_schema(&self) -> Result { + match self.children().next() { + None => Ok(DataSchemaRef::default()), + Some(child) => child.output_schema() + } + } + + fn children<'a>(&'a self) -> Box> + 'a> { + Box::new(std::iter::empty()) + } + + fn children_mut<'a>(&'a self) -> Box> + 'a> { + Box::new(std::iter::empty()) + } + + #[recursive::recursive] + fn try_find_mutation_source(&self) -> Option { + for child in self.children() { + if let Some(plan) = child.try_find_mutation_source() { + return Some(plan); + } + } + + None + } + + #[recursive::recursive] + fn get_all_data_source(&self, sources: &mut Vec<(u32, Box)>) { + for child in self.children() { + child.get_all_data_source(sources); + } + } + + #[recursive::recursive] + fn set_pruning_stats(&mut self, stats: &mut HashMap) { + for child in self.children_mut() { + child.set_pruning_stats(stats) + } + } + + #[recursive::recursive] + fn is_distributed_plan(&self) -> bool { + self.children().any(|child| child.is_distributed_plan()) + // || matches!( + // self, + // Self::ExchangeSource(_) | Self::ExchangeSink(_) | Self::Exchange(_) + // ) + } + + #[recursive::recursive] + fn is_warehouse_distributed_plan(&self) -> bool { + self.children() + .any(|child| child.is_warehouse_distributed_plan()) + } +} + #[derive(serde::Serialize, serde::Deserialize, Educe, EnumAsInner)] #[educe( Clone(bound = false, attrs = "#[recursive::recursive]"), @@ -167,321 +278,321 @@ impl PhysicalPlan { /// Adjust the plan_id of the physical plan. /// This function will assign a unique plan_id to each physical plan node in a top-down manner. /// Which means the plan_id of a node is always greater than the plan_id of its parent node. - #[recursive::recursive] - pub fn adjust_plan_id(&mut self, next_id: &mut u32) { - match self { - PhysicalPlan::AsyncFunction(plan) => { - plan.plan_id = *next_id; - *next_id += 1; - } - PhysicalPlan::TableScan(plan) => { - plan.plan_id = *next_id; - *next_id += 1; - } - PhysicalPlan::Filter(plan) => { - plan.plan_id = *next_id; - *next_id += 1; - plan.input.adjust_plan_id(next_id); - } - PhysicalPlan::EvalScalar(plan) => { - plan.plan_id = *next_id; - *next_id += 1; - plan.input.adjust_plan_id(next_id); - } - PhysicalPlan::ProjectSet(plan) => { - plan.plan_id = *next_id; - *next_id += 1; - plan.input.adjust_plan_id(next_id); - } - PhysicalPlan::AggregateExpand(plan) => { - plan.plan_id = *next_id; - *next_id += 1; - plan.input.adjust_plan_id(next_id); - } - PhysicalPlan::AggregatePartial(plan) => { - plan.plan_id = *next_id; - *next_id += 1; - plan.input.adjust_plan_id(next_id); - } - PhysicalPlan::AggregateFinal(plan) => { - plan.plan_id = *next_id; - *next_id += 1; - plan.input.adjust_plan_id(next_id); - } - PhysicalPlan::Window(plan) => { - plan.plan_id = *next_id; - *next_id += 1; - plan.input.adjust_plan_id(next_id); - } - PhysicalPlan::WindowPartition(plan) => { - plan.plan_id = *next_id; - *next_id += 1; - plan.input.adjust_plan_id(next_id); - } - PhysicalPlan::Sort(plan) => { - plan.plan_id = *next_id; - *next_id += 1; - plan.input.adjust_plan_id(next_id); - } - PhysicalPlan::Limit(plan) => { - plan.plan_id = *next_id; - *next_id += 1; - plan.input.adjust_plan_id(next_id); - } - PhysicalPlan::RowFetch(plan) => { - plan.plan_id = *next_id; - *next_id += 1; - plan.input.adjust_plan_id(next_id); - } - PhysicalPlan::HashJoin(plan) => { - plan.plan_id = *next_id; - *next_id += 1; - plan.probe.adjust_plan_id(next_id); - plan.build.adjust_plan_id(next_id); - } - PhysicalPlan::RangeJoin(plan) => { - plan.plan_id = *next_id; - *next_id += 1; - plan.left.adjust_plan_id(next_id); - plan.right.adjust_plan_id(next_id); - } - PhysicalPlan::Exchange(plan) => { - plan.plan_id = *next_id; - *next_id += 1; - plan.input.adjust_plan_id(next_id); - } - PhysicalPlan::UnionAll(plan) => { - plan.plan_id = *next_id; - *next_id += 1; - plan.left.adjust_plan_id(next_id); - plan.right.adjust_plan_id(next_id); - } - PhysicalPlan::RecursiveCteScan(plan) => { - plan.plan_id = *next_id; - *next_id += 1; - } - PhysicalPlan::ConstantTableScan(plan) => { - plan.plan_id = *next_id; - *next_id += 1; - } - PhysicalPlan::ExpressionScan(plan) => { - plan.plan_id = *next_id; - *next_id += 1; - } - PhysicalPlan::CacheScan(plan) => { - plan.plan_id = *next_id; - *next_id += 1; - } - PhysicalPlan::Udf(plan) => { - plan.plan_id = *next_id; - *next_id += 1; - plan.input.adjust_plan_id(next_id); - } - PhysicalPlan::DistributedInsertSelect(plan) => { - plan.plan_id = *next_id; - *next_id += 1; - plan.input.adjust_plan_id(next_id); - } - PhysicalPlan::ExchangeSource(plan) => { - plan.plan_id = *next_id; - *next_id += 1; - } - PhysicalPlan::ExchangeSink(plan) => { - plan.plan_id = *next_id; - *next_id += 1; - } - PhysicalPlan::CopyIntoTable(plan) => { - plan.plan_id = *next_id; - *next_id += 1; - match &mut plan.source { - CopyIntoTableSource::Query(input) => input.adjust_plan_id(next_id), - CopyIntoTableSource::Stage(input) => input.adjust_plan_id(next_id), - }; - } - PhysicalPlan::CopyIntoLocation(plan) => { - plan.plan_id = *next_id; - *next_id += 1; - plan.input.adjust_plan_id(next_id); - } - PhysicalPlan::ReplaceInto(plan) => { - plan.plan_id = *next_id; - *next_id += 1; - plan.input.adjust_plan_id(next_id); - } - PhysicalPlan::MutationSource(plan) => { - plan.plan_id = *next_id; - *next_id += 1; - } - PhysicalPlan::ColumnMutation(plan) => { - plan.plan_id = *next_id; - *next_id += 1; - plan.input.adjust_plan_id(next_id); - } - PhysicalPlan::Mutation(plan) => { - plan.plan_id = *next_id; - *next_id += 1; - plan.input.adjust_plan_id(next_id); - } - PhysicalPlan::MutationSplit(plan) => { - plan.plan_id = *next_id; - *next_id += 1; - plan.input.adjust_plan_id(next_id); - } - PhysicalPlan::MutationManipulate(plan) => { - plan.plan_id = *next_id; - *next_id += 1; - plan.input.adjust_plan_id(next_id); - } - PhysicalPlan::MutationOrganize(plan) => { - plan.plan_id = *next_id; - *next_id += 1; - plan.input.adjust_plan_id(next_id); - } - PhysicalPlan::AddStreamColumn(plan) => { - plan.plan_id = *next_id; - *next_id += 1; - plan.input.adjust_plan_id(next_id); - } - PhysicalPlan::CommitSink(plan) => { - plan.plan_id = *next_id; - *next_id += 1; - plan.input.adjust_plan_id(next_id); - } - PhysicalPlan::ReplaceAsyncSourcer(plan) => { - plan.plan_id = *next_id; - *next_id += 1; - } - PhysicalPlan::ReplaceDeduplicate(plan) => { - plan.plan_id = *next_id; - *next_id += 1; - plan.input.adjust_plan_id(next_id); - } - PhysicalPlan::CompactSource(plan) => { - plan.plan_id = *next_id; - *next_id += 1; - } - PhysicalPlan::Recluster(plan) => { - plan.plan_id = *next_id; - *next_id += 1; - } - PhysicalPlan::HilbertPartition(plan) => { - plan.plan_id = *next_id; - *next_id += 1; - } - PhysicalPlan::Duplicate(plan) => { - plan.plan_id = *next_id; - *next_id += 1; - plan.input.adjust_plan_id(next_id); - } - PhysicalPlan::Shuffle(plan) => { - plan.plan_id = *next_id; - *next_id += 1; - plan.input.adjust_plan_id(next_id); - } - PhysicalPlan::ChunkFilter(plan) => { - plan.plan_id = *next_id; - *next_id += 1; - plan.input.adjust_plan_id(next_id); - } - PhysicalPlan::ChunkEvalScalar(plan) => { - plan.plan_id = *next_id; - *next_id += 1; - plan.input.adjust_plan_id(next_id); - } - PhysicalPlan::ChunkCastSchema(plan) => { - plan.plan_id = *next_id; - *next_id += 1; - plan.input.adjust_plan_id(next_id); - } - PhysicalPlan::ChunkFillAndReorder(plan) => { - plan.plan_id = *next_id; - *next_id += 1; - plan.input.adjust_plan_id(next_id); - } - PhysicalPlan::ChunkAppendData(plan) => { - plan.plan_id = *next_id; - *next_id += 1; - plan.input.adjust_plan_id(next_id); - } - PhysicalPlan::ChunkMerge(plan) => { - plan.plan_id = *next_id; - *next_id += 1; - plan.input.adjust_plan_id(next_id); - } - PhysicalPlan::ChunkCommitInsert(plan) => { - plan.plan_id = *next_id; - *next_id += 1; - plan.input.adjust_plan_id(next_id); - } - PhysicalPlan::BroadcastSource(plan) => { - plan.plan_id = *next_id; - *next_id += 1; - } - PhysicalPlan::BroadcastSink(plan) => { - plan.plan_id = *next_id; - *next_id += 1; - plan.input.adjust_plan_id(next_id); - } - } - } + // #[recursive::recursive] + // pub fn adjust_plan_id(&mut self, next_id: &mut u32) { + // match self { + // PhysicalPlan::AsyncFunction(plan) => { + // plan.plan_id = *next_id; + // *next_id += 1; + // } + // PhysicalPlan::TableScan(plan) => { + // plan.plan_id = *next_id; + // *next_id += 1; + // } + // PhysicalPlan::Filter(plan) => { + // plan.plan_id = *next_id; + // *next_id += 1; + // plan.input.adjust_plan_id(next_id); + // } + // PhysicalPlan::EvalScalar(plan) => { + // plan.plan_id = *next_id; + // *next_id += 1; + // plan.input.adjust_plan_id(next_id); + // } + // PhysicalPlan::ProjectSet(plan) => { + // plan.plan_id = *next_id; + // *next_id += 1; + // plan.input.adjust_plan_id(next_id); + // } + // PhysicalPlan::AggregateExpand(plan) => { + // plan.plan_id = *next_id; + // *next_id += 1; + // plan.input.adjust_plan_id(next_id); + // } + // PhysicalPlan::AggregatePartial(plan) => { + // plan.plan_id = *next_id; + // *next_id += 1; + // plan.input.adjust_plan_id(next_id); + // } + // PhysicalPlan::AggregateFinal(plan) => { + // plan.plan_id = *next_id; + // *next_id += 1; + // plan.input.adjust_plan_id(next_id); + // } + // PhysicalPlan::Window(plan) => { + // plan.plan_id = *next_id; + // *next_id += 1; + // plan.input.adjust_plan_id(next_id); + // } + // PhysicalPlan::WindowPartition(plan) => { + // plan.plan_id = *next_id; + // *next_id += 1; + // plan.input.adjust_plan_id(next_id); + // } + // PhysicalPlan::Sort(plan) => { + // plan.plan_id = *next_id; + // *next_id += 1; + // plan.input.adjust_plan_id(next_id); + // } + // PhysicalPlan::Limit(plan) => { + // plan.plan_id = *next_id; + // *next_id += 1; + // plan.input.adjust_plan_id(next_id); + // } + // PhysicalPlan::RowFetch(plan) => { + // plan.plan_id = *next_id; + // *next_id += 1; + // plan.input.adjust_plan_id(next_id); + // } + // PhysicalPlan::HashJoin(plan) => { + // plan.plan_id = *next_id; + // *next_id += 1; + // plan.probe.adjust_plan_id(next_id); + // plan.build.adjust_plan_id(next_id); + // } + // PhysicalPlan::RangeJoin(plan) => { + // plan.plan_id = *next_id; + // *next_id += 1; + // plan.left.adjust_plan_id(next_id); + // plan.right.adjust_plan_id(next_id); + // } + // PhysicalPlan::Exchange(plan) => { + // plan.plan_id = *next_id; + // *next_id += 1; + // plan.input.adjust_plan_id(next_id); + // } + // PhysicalPlan::UnionAll(plan) => { + // plan.plan_id = *next_id; + // *next_id += 1; + // plan.left.adjust_plan_id(next_id); + // plan.right.adjust_plan_id(next_id); + // } + // PhysicalPlan::RecursiveCteScan(plan) => { + // plan.plan_id = *next_id; + // *next_id += 1; + // } + // PhysicalPlan::ConstantTableScan(plan) => { + // plan.plan_id = *next_id; + // *next_id += 1; + // } + // PhysicalPlan::ExpressionScan(plan) => { + // plan.plan_id = *next_id; + // *next_id += 1; + // } + // PhysicalPlan::CacheScan(plan) => { + // plan.plan_id = *next_id; + // *next_id += 1; + // } + // PhysicalPlan::Udf(plan) => { + // plan.plan_id = *next_id; + // *next_id += 1; + // plan.input.adjust_plan_id(next_id); + // } + // PhysicalPlan::DistributedInsertSelect(plan) => { + // plan.plan_id = *next_id; + // *next_id += 1; + // plan.input.adjust_plan_id(next_id); + // } + // PhysicalPlan::ExchangeSource(plan) => { + // plan.plan_id = *next_id; + // *next_id += 1; + // } + // PhysicalPlan::ExchangeSink(plan) => { + // plan.plan_id = *next_id; + // *next_id += 1; + // } + // PhysicalPlan::CopyIntoTable(plan) => { + // plan.plan_id = *next_id; + // *next_id += 1; + // match &mut plan.source { + // CopyIntoTableSource::Query(input) => input.adjust_plan_id(next_id), + // CopyIntoTableSource::Stage(input) => input.adjust_plan_id(next_id), + // }; + // } + // PhysicalPlan::CopyIntoLocation(plan) => { + // plan.plan_id = *next_id; + // *next_id += 1; + // plan.input.adjust_plan_id(next_id); + // } + // PhysicalPlan::ReplaceInto(plan) => { + // plan.plan_id = *next_id; + // *next_id += 1; + // plan.input.adjust_plan_id(next_id); + // } + // PhysicalPlan::MutationSource(plan) => { + // plan.plan_id = *next_id; + // *next_id += 1; + // } + // PhysicalPlan::ColumnMutation(plan) => { + // plan.plan_id = *next_id; + // *next_id += 1; + // plan.input.adjust_plan_id(next_id); + // } + // PhysicalPlan::Mutation(plan) => { + // plan.plan_id = *next_id; + // *next_id += 1; + // plan.input.adjust_plan_id(next_id); + // } + // PhysicalPlan::MutationSplit(plan) => { + // plan.plan_id = *next_id; + // *next_id += 1; + // plan.input.adjust_plan_id(next_id); + // } + // PhysicalPlan::MutationManipulate(plan) => { + // plan.plan_id = *next_id; + // *next_id += 1; + // plan.input.adjust_plan_id(next_id); + // } + // PhysicalPlan::MutationOrganize(plan) => { + // plan.plan_id = *next_id; + // *next_id += 1; + // plan.input.adjust_plan_id(next_id); + // } + // PhysicalPlan::AddStreamColumn(plan) => { + // plan.plan_id = *next_id; + // *next_id += 1; + // plan.input.adjust_plan_id(next_id); + // } + // PhysicalPlan::CommitSink(plan) => { + // plan.plan_id = *next_id; + // *next_id += 1; + // plan.input.adjust_plan_id(next_id); + // } + // PhysicalPlan::ReplaceAsyncSourcer(plan) => { + // plan.plan_id = *next_id; + // *next_id += 1; + // } + // PhysicalPlan::ReplaceDeduplicate(plan) => { + // plan.plan_id = *next_id; + // *next_id += 1; + // plan.input.adjust_plan_id(next_id); + // } + // PhysicalPlan::CompactSource(plan) => { + // plan.plan_id = *next_id; + // *next_id += 1; + // } + // PhysicalPlan::Recluster(plan) => { + // plan.plan_id = *next_id; + // *next_id += 1; + // } + // PhysicalPlan::HilbertPartition(plan) => { + // plan.plan_id = *next_id; + // *next_id += 1; + // } + // PhysicalPlan::Duplicate(plan) => { + // plan.plan_id = *next_id; + // *next_id += 1; + // plan.input.adjust_plan_id(next_id); + // } + // PhysicalPlan::Shuffle(plan) => { + // plan.plan_id = *next_id; + // *next_id += 1; + // plan.input.adjust_plan_id(next_id); + // } + // PhysicalPlan::ChunkFilter(plan) => { + // plan.plan_id = *next_id; + // *next_id += 1; + // plan.input.adjust_plan_id(next_id); + // } + // PhysicalPlan::ChunkEvalScalar(plan) => { + // plan.plan_id = *next_id; + // *next_id += 1; + // plan.input.adjust_plan_id(next_id); + // } + // PhysicalPlan::ChunkCastSchema(plan) => { + // plan.plan_id = *next_id; + // *next_id += 1; + // plan.input.adjust_plan_id(next_id); + // } + // PhysicalPlan::ChunkFillAndReorder(plan) => { + // plan.plan_id = *next_id; + // *next_id += 1; + // plan.input.adjust_plan_id(next_id); + // } + // PhysicalPlan::ChunkAppendData(plan) => { + // plan.plan_id = *next_id; + // *next_id += 1; + // plan.input.adjust_plan_id(next_id); + // } + // PhysicalPlan::ChunkMerge(plan) => { + // plan.plan_id = *next_id; + // *next_id += 1; + // plan.input.adjust_plan_id(next_id); + // } + // PhysicalPlan::ChunkCommitInsert(plan) => { + // plan.plan_id = *next_id; + // *next_id += 1; + // plan.input.adjust_plan_id(next_id); + // } + // PhysicalPlan::BroadcastSource(plan) => { + // plan.plan_id = *next_id; + // *next_id += 1; + // } + // PhysicalPlan::BroadcastSink(plan) => { + // plan.plan_id = *next_id; + // *next_id += 1; + // plan.input.adjust_plan_id(next_id); + // } + // } + // } /// Get the id of the plan node - pub fn get_id(&self) -> u32 { - match self { - PhysicalPlan::AsyncFunction(v) => v.plan_id, - PhysicalPlan::TableScan(v) => v.plan_id, - PhysicalPlan::Filter(v) => v.plan_id, - PhysicalPlan::EvalScalar(v) => v.plan_id, - PhysicalPlan::ProjectSet(v) => v.plan_id, - PhysicalPlan::AggregateExpand(v) => v.plan_id, - PhysicalPlan::AggregatePartial(v) => v.plan_id, - PhysicalPlan::AggregateFinal(v) => v.plan_id, - PhysicalPlan::Window(v) => v.plan_id, - PhysicalPlan::WindowPartition(v) => v.plan_id, - PhysicalPlan::Sort(v) => v.plan_id, - PhysicalPlan::Limit(v) => v.plan_id, - PhysicalPlan::RowFetch(v) => v.plan_id, - PhysicalPlan::HashJoin(v) => v.plan_id, - PhysicalPlan::RangeJoin(v) => v.plan_id, - PhysicalPlan::Exchange(v) => v.plan_id, - PhysicalPlan::UnionAll(v) => v.plan_id, - PhysicalPlan::DistributedInsertSelect(v) => v.plan_id, - PhysicalPlan::ExchangeSource(v) => v.plan_id, - PhysicalPlan::ExchangeSink(v) => v.plan_id, - PhysicalPlan::ConstantTableScan(v) => v.plan_id, - PhysicalPlan::ExpressionScan(v) => v.plan_id, - PhysicalPlan::CacheScan(v) => v.plan_id, - PhysicalPlan::Udf(v) => v.plan_id, - PhysicalPlan::MutationSource(v) => v.plan_id, - PhysicalPlan::ColumnMutation(v) => v.plan_id, - PhysicalPlan::Mutation(v) => v.plan_id, - PhysicalPlan::MutationSplit(v) => v.plan_id, - PhysicalPlan::MutationManipulate(v) => v.plan_id, - PhysicalPlan::MutationOrganize(v) => v.plan_id, - PhysicalPlan::AddStreamColumn(v) => v.plan_id, - PhysicalPlan::CommitSink(v) => v.plan_id, - PhysicalPlan::CopyIntoTable(v) => v.plan_id, - PhysicalPlan::CopyIntoLocation(v) => v.plan_id, - PhysicalPlan::ReplaceAsyncSourcer(v) => v.plan_id, - PhysicalPlan::ReplaceDeduplicate(v) => v.plan_id, - PhysicalPlan::ReplaceInto(v) => v.plan_id, - PhysicalPlan::CompactSource(v) => v.plan_id, - PhysicalPlan::Recluster(v) => v.plan_id, - PhysicalPlan::HilbertPartition(v) => v.plan_id, - PhysicalPlan::Duplicate(v) => v.plan_id, - PhysicalPlan::Shuffle(v) => v.plan_id, - PhysicalPlan::ChunkFilter(v) => v.plan_id, - PhysicalPlan::ChunkEvalScalar(v) => v.plan_id, - PhysicalPlan::ChunkCastSchema(v) => v.plan_id, - PhysicalPlan::ChunkFillAndReorder(v) => v.plan_id, - PhysicalPlan::ChunkAppendData(v) => v.plan_id, - PhysicalPlan::ChunkMerge(v) => v.plan_id, - PhysicalPlan::ChunkCommitInsert(v) => v.plan_id, - PhysicalPlan::RecursiveCteScan(v) => v.plan_id, - PhysicalPlan::BroadcastSource(v) => v.plan_id, - PhysicalPlan::BroadcastSink(v) => v.plan_id, - } - } + // pub fn get_id(&self) -> u32 { + // match self { + // PhysicalPlan::AsyncFunction(v) => v.plan_id, + // PhysicalPlan::TableScan(v) => v.plan_id, + // PhysicalPlan::Filter(v) => v.plan_id, + // PhysicalPlan::EvalScalar(v) => v.plan_id, + // PhysicalPlan::ProjectSet(v) => v.plan_id, + // PhysicalPlan::AggregateExpand(v) => v.plan_id, + // PhysicalPlan::AggregatePartial(v) => v.plan_id, + // PhysicalPlan::AggregateFinal(v) => v.plan_id, + // PhysicalPlan::Window(v) => v.plan_id, + // PhysicalPlan::WindowPartition(v) => v.plan_id, + // PhysicalPlan::Sort(v) => v.plan_id, + // PhysicalPlan::Limit(v) => v.plan_id, + // PhysicalPlan::RowFetch(v) => v.plan_id, + // PhysicalPlan::HashJoin(v) => v.plan_id, + // PhysicalPlan::RangeJoin(v) => v.plan_id, + // PhysicalPlan::Exchange(v) => v.plan_id, + // PhysicalPlan::UnionAll(v) => v.plan_id, + // PhysicalPlan::DistributedInsertSelect(v) => v.plan_id, + // PhysicalPlan::ExchangeSource(v) => v.plan_id, + // PhysicalPlan::ExchangeSink(v) => v.plan_id, + // PhysicalPlan::ConstantTableScan(v) => v.plan_id, + // PhysicalPlan::ExpressionScan(v) => v.plan_id, + // PhysicalPlan::CacheScan(v) => v.plan_id, + // PhysicalPlan::Udf(v) => v.plan_id, + // PhysicalPlan::MutationSource(v) => v.plan_id, + // PhysicalPlan::ColumnMutation(v) => v.plan_id, + // PhysicalPlan::Mutation(v) => v.plan_id, + // PhysicalPlan::MutationSplit(v) => v.plan_id, + // PhysicalPlan::MutationManipulate(v) => v.plan_id, + // PhysicalPlan::MutationOrganize(v) => v.plan_id, + // PhysicalPlan::AddStreamColumn(v) => v.plan_id, + // PhysicalPlan::CommitSink(v) => v.plan_id, + // PhysicalPlan::CopyIntoTable(v) => v.plan_id, + // PhysicalPlan::CopyIntoLocation(v) => v.plan_id, + // PhysicalPlan::ReplaceAsyncSourcer(v) => v.plan_id, + // PhysicalPlan::ReplaceDeduplicate(v) => v.plan_id, + // PhysicalPlan::ReplaceInto(v) => v.plan_id, + // PhysicalPlan::CompactSource(v) => v.plan_id, + // PhysicalPlan::Recluster(v) => v.plan_id, + // PhysicalPlan::HilbertPartition(v) => v.plan_id, + // PhysicalPlan::Duplicate(v) => v.plan_id, + // PhysicalPlan::Shuffle(v) => v.plan_id, + // PhysicalPlan::ChunkFilter(v) => v.plan_id, + // PhysicalPlan::ChunkEvalScalar(v) => v.plan_id, + // PhysicalPlan::ChunkCastSchema(v) => v.plan_id, + // PhysicalPlan::ChunkFillAndReorder(v) => v.plan_id, + // PhysicalPlan::ChunkAppendData(v) => v.plan_id, + // PhysicalPlan::ChunkMerge(v) => v.plan_id, + // PhysicalPlan::ChunkCommitInsert(v) => v.plan_id, + // PhysicalPlan::RecursiveCteScan(v) => v.plan_id, + // PhysicalPlan::BroadcastSource(v) => v.plan_id, + // PhysicalPlan::BroadcastSink(v) => v.plan_id, + // } + // } pub fn output_schema(&self) -> Result { match self { @@ -603,7 +714,7 @@ impl PhysicalPlan { } } - pub fn children<'a>(&'a self) -> Box + 'a> { + pub fn children<'a>(&'a self) -> Box + 'a> { match self { PhysicalPlan::TableScan(_) | PhysicalPlan::ConstantTableScan(_) @@ -677,7 +788,7 @@ impl PhysicalPlan { } } - pub fn children_mut<'a>(&'a mut self) -> Box + 'a> { + pub fn children_mut<'a>(&'a mut self) -> Box + 'a> { match self { PhysicalPlan::TableScan(_) | PhysicalPlan::ConstantTableScan(_) @@ -752,6 +863,7 @@ impl PhysicalPlan { } /// Used to find data source info in a non-aggregation and single-table query plan. + #[recursive::recursive] pub fn try_find_single_data_source(&self) -> Option<&DataSourcePlan> { match self { PhysicalPlan::TableScan(scan) => Some(&scan.source), @@ -818,12 +930,12 @@ impl PhysicalPlan { ) } - #[recursive::recursive] - pub fn is_warehouse_distributed_plan(&self) -> bool { - self.children() - .any(|child| child.is_warehouse_distributed_plan()) - || matches!(self, Self::TableScan(v) if v.source.parts.kind == PartitionsShuffleKind::BroadcastWarehouse) - } + // #[recursive::recursive] + // pub fn is_warehouse_distributed_plan(&self) -> bool { + // self.children() + // .any(|child| child.is_warehouse_distributed_plan()) + // || matches!(self, Self::TableScan(v) if v.source.parts.kind == PartitionsShuffleKind::BroadcastWarehouse) + // } pub fn get_desc(&self) -> Result { Ok(match self { @@ -1108,32 +1220,32 @@ impl PhysicalPlan { } } - #[recursive::recursive] - pub fn get_all_data_source(&self, sources: &mut Vec<(u32, Box)>) { - match self { - PhysicalPlan::TableScan(table_scan) => { - sources.push((table_scan.plan_id, table_scan.source.clone())); - } - _ => { - for child in self.children() { - child.get_all_data_source(sources); - } - } - } - } - - pub fn set_pruning_stats(&mut self, stats: &mut HashMap) { - match self { - PhysicalPlan::TableScan(table_scan) => { - if let Some(stat) = stats.remove(&table_scan.plan_id) { - table_scan.source.statistics = stat; - } - } - _ => { - for child in self.children_mut() { - child.set_pruning_stats(stats) - } - } - } - } + // #[recursive::recursive] + // pub fn get_all_data_source(&self, sources: &mut Vec<(u32, Box)>) { + // match self { + // PhysicalPlan::TableScan(table_scan) => { + // sources.push((table_scan.plan_id, table_scan.source.clone())); + // } + // _ => { + // for child in self.children() { + // child.get_all_data_source(sources); + // } + // } + // } + // } + // + // pub fn set_pruning_stats(&mut self, stats: &mut HashMap) { + // match self { + // PhysicalPlan::TableScan(table_scan) => { + // if let Some(stat) = stats.remove(&table_scan.plan_id) { + // table_scan.source.statistics = stat; + // } + // } + // _ => { + // for child in self.children_mut() { + // child.set_pruning_stats(stats) + // } + // } + // } + // } } diff --git a/src/query/sql/src/executor/physical_plans/physical_add_stream_column.rs b/src/query/sql/src/executor/physical_plans/physical_add_stream_column.rs index c08bb9660fe99..3e140a596253c 100644 --- a/src/query/sql/src/executor/physical_plans/physical_add_stream_column.rs +++ b/src/query/sql/src/executor/physical_plans/physical_add_stream_column.rs @@ -23,7 +23,7 @@ use databend_common_expression::ORIGIN_BLOCK_ID_COL_NAME; use databend_common_expression::ORIGIN_BLOCK_ROW_NUM_COL_NAME; use databend_common_expression::ORIGIN_VERSION_COL_NAME; -use crate::executor::PhysicalPlan; +use crate::executor::{IPhysicalPlan, PhysicalPlan}; use crate::planner::CURRENT_BLOCK_ID_COL_NAME; use crate::planner::CURRENT_BLOCK_ROW_NUM_COL_NAME; use crate::plans::BoundColumnRef; @@ -44,6 +44,10 @@ pub struct AddStreamColumn { pub stream_columns: Vec, } +impl IPhysicalPlan for AddStreamColumn { + +} + impl AddStreamColumn { pub fn new( metadata: &MetadataRef, diff --git a/src/query/sql/src/executor/physical_plans/physical_aggregate_expand.rs b/src/query/sql/src/executor/physical_plans/physical_aggregate_expand.rs index a9ea7055912cd..69db39f6daf0a 100644 --- a/src/query/sql/src/executor/physical_plans/physical_aggregate_expand.rs +++ b/src/query/sql/src/executor/physical_plans/physical_aggregate_expand.rs @@ -20,7 +20,7 @@ use databend_common_expression::DataSchemaRef; use databend_common_expression::DataSchemaRefExt; use crate::executor::explain::PlanStatsInfo; -use crate::executor::PhysicalPlan; +use crate::executor::{IPhysicalPlan, PhysicalPlan, PhysicalPlanMeta}; use crate::plans::GroupingSets; use crate::IndexType; @@ -29,7 +29,8 @@ use crate::IndexType; pub struct AggregateExpand { // A unique id of operator in a `PhysicalPlan` tree, only used for display. pub plan_id: u32, - pub input: Box, + meta: PhysicalPlanMeta, + pub input: Box, pub group_bys: Vec, pub grouping_sets: GroupingSets, @@ -37,6 +38,51 @@ pub struct AggregateExpand { pub stat_info: Option, } +impl IPhysicalPlan for AggregateExpand { + fn get_meta(&self) -> &PhysicalPlanMeta { + &self.meta + } + + fn get_meta_mut(&mut self) -> &mut PhysicalPlanMeta { + &mut self.meta + } + + fn output_schema(&self) -> Result { + let input_schema = self.input.output_schema()?; + let mut output_fields = input_schema.fields().clone(); + // Add virtual columns to group by. + output_fields.reserve(self.group_bys.len() + 1); + + for (group_by, (actual, ty)) in self + .group_bys + .iter() + .zip(self.grouping_sets.dup_group_items.iter()) + { + // All group by columns will wrap nullable. + let i = input_schema.index_of(&group_by.to_string())?; + let f = &mut output_fields[i]; + debug_assert!(f.data_type() == ty || f.data_type().wrap_nullable() == *ty); + *f = DataField::new(f.name(), f.data_type().wrap_nullable()); + let new_field = DataField::new(&actual.to_string(), ty.clone()); + output_fields.push(new_field); + } + + output_fields.push(DataField::new( + &self.grouping_sets.grouping_id_index.to_string(), + DataType::Number(NumberDataType::UInt32), + )); + Ok(DataSchemaRefExt::create(output_fields)) + } + + fn children<'a>(&'a self) -> Box> + 'a> { + Box::new(std::iter::once(&self.input)) + } + + fn children_mut<'a>(&'a self) -> Box> + 'a> { + Box::new(std::iter::once(&mut self.input)) + } +} + impl AggregateExpand { pub fn output_schema(&self) -> Result { let input_schema = self.input.output_schema()?; diff --git a/src/query/sql/src/executor/physical_plans/physical_aggregate_final.rs b/src/query/sql/src/executor/physical_plans/physical_aggregate_final.rs index d3511fbb8cdaa..af38893493552 100644 --- a/src/query/sql/src/executor/physical_plans/physical_aggregate_final.rs +++ b/src/query/sql/src/executor/physical_plans/physical_aggregate_final.rs @@ -28,7 +28,7 @@ use crate::executor::physical_plans::AggregateFunctionDesc; use crate::executor::physical_plans::AggregateFunctionSignature; use crate::executor::physical_plans::AggregatePartial; use crate::executor::physical_plans::Exchange; -use crate::executor::PhysicalPlan; +use crate::executor::{IPhysicalPlan, PhysicalPlan, PhysicalPlanMeta}; use crate::executor::PhysicalPlanBuilder; use crate::optimizer::ir::SExpr; use crate::plans::AggregateMode; @@ -39,9 +39,10 @@ use crate::ScalarExpr; #[derive(serde::Serialize, serde::Deserialize, Clone, Debug)] pub struct AggregateFinal { + meta: PhysicalPlanMeta, // A unique id of operator in a `PhysicalPlan` tree, only used for display. pub plan_id: u32, - pub input: Box, + pub input: Box, pub group_by: Vec, pub agg_funcs: Vec, pub before_group_by_schema: DataSchemaRef, @@ -51,6 +52,41 @@ pub struct AggregateFinal { pub stat_info: Option, } +impl IPhysicalPlan for AggregateFinal { + fn get_meta(&self) -> &PhysicalPlanMeta { + &self.meta + } + + fn get_meta_mut(&mut self) -> &mut PhysicalPlanMeta { + &mut self.meta + } + + fn output_schema(&self) -> Result { + let mut fields = Vec::with_capacity(self.agg_funcs.len() + self.group_by.len()); + for agg in self.agg_funcs.iter() { + let data_type = agg.sig.return_type.clone(); + fields.push(DataField::new(&agg.output_column.to_string(), data_type)); + } + for id in self.group_by.iter() { + let data_type = self + .before_group_by_schema + .field_with_name(&id.to_string())? + .data_type() + .clone(); + fields.push(DataField::new(&id.to_string(), data_type)); + } + Ok(DataSchemaRefExt::create(fields)) + } + + fn children<'a>(&'a self) -> Box> + 'a> { + Box::new(std::iter::once(&self.input)) + } + + fn children_mut<'a>(&'a self) -> Box> + 'a> { + Box::new(std::iter::once(&mut self.input)) + } +} + impl AggregateFinal { pub fn output_schema(&self) -> Result { let mut fields = Vec::with_capacity(self.agg_funcs.len() + self.group_by.len()); diff --git a/src/query/sql/src/executor/physical_plans/physical_aggregate_partial.rs b/src/query/sql/src/executor/physical_plans/physical_aggregate_partial.rs index a8a73071aaa57..7727a5889f18f 100644 --- a/src/query/sql/src/executor/physical_plans/physical_aggregate_partial.rs +++ b/src/query/sql/src/executor/physical_plans/physical_aggregate_partial.rs @@ -23,11 +23,12 @@ use databend_common_expression::DataSchemaRefExt; use super::SortDesc; use crate::executor::explain::PlanStatsInfo; use crate::executor::physical_plans::common::AggregateFunctionDesc; -use crate::executor::PhysicalPlan; +use crate::executor::{IPhysicalPlan, PhysicalPlan, PhysicalPlanMeta}; use crate::IndexType; #[derive(serde::Serialize, serde::Deserialize, Clone, Debug)] pub struct AggregatePartial { + meta: PhysicalPlanMeta, // A unique id of operator in a `PhysicalPlan` tree, only used for display. pub plan_id: u32, pub input: Box, @@ -42,6 +43,45 @@ pub struct AggregatePartial { pub stat_info: Option, } +impl IPhysicalPlan for AggregatePartial { + fn get_meta(&self) -> &PhysicalPlanMeta { + &self.meta + } + + fn get_meta_mut(&mut self) -> &mut PhysicalPlanMeta { + &mut self.meta + } + + fn output_schema(&self) -> Result { + let input_schema = self.input.output_schema()?; + + let mut fields = Vec::with_capacity(self.agg_funcs.len() + self.group_by.len()); + + fields.extend(self.agg_funcs.iter().map(|func| { + let name = func.output_column.to_string(); + DataField::new(&name, DataType::Binary) + })); + + for (idx, field) in self.group_by.iter().zip( + self.group_by + .iter() + .map(|index| input_schema.field_with_name(&index.to_string())), + ) { + fields.push(DataField::new(&idx.to_string(), field?.data_type().clone())); + } + + Ok(DataSchemaRefExt::create(fields)) + } + + fn children<'a>(&'a self) -> Box> + 'a> { + Box::new(std::iter::once(&self.input)) + } + + fn children_mut<'a>(&'a self) -> Box> + 'a> { + Box::new(std::iter::once(&mut self.input)) + } +} + impl AggregatePartial { pub fn output_schema(&self) -> Result { let input_schema = self.input.output_schema()?; diff --git a/src/query/sql/src/executor/physical_plans/physical_async_func.rs b/src/query/sql/src/executor/physical_plans/physical_async_func.rs index 503a90dfac72f..05c43a209ddf7 100644 --- a/src/query/sql/src/executor/physical_plans/physical_async_func.rs +++ b/src/query/sql/src/executor/physical_plans/physical_async_func.rs @@ -20,7 +20,7 @@ use databend_common_expression::DataSchemaRef; use databend_common_expression::DataSchemaRefExt; use crate::executor::explain::PlanStatsInfo; -use crate::executor::PhysicalPlan; +use crate::executor::{IPhysicalPlan, PhysicalPlan}; use crate::executor::PhysicalPlanBuilder; use crate::optimizer::ir::SExpr; use crate::plans::AsyncFunctionArgument; @@ -38,6 +38,19 @@ pub struct AsyncFunction { pub stat_info: Option, } +impl IPhysicalPlan for AsyncFunction { + fn output_schema(&self) -> Result { + let input_schema = self.input.output_schema()?; + let mut fields = input_schema.fields().clone(); + for async_func_desc in self.async_func_descs.iter() { + let name = async_func_desc.output_column.to_string(); + let data_type = async_func_desc.data_type.clone(); + fields.push(DataField::new(&name, *data_type)); + } + Ok(DataSchemaRefExt::create(fields)) + } +} + impl AsyncFunction { pub fn output_schema(&self) -> Result { let input_schema = self.input.output_schema()?; diff --git a/src/query/sql/src/executor/physical_plans/physical_broadcast.rs b/src/query/sql/src/executor/physical_plans/physical_broadcast.rs index 36e42243e4ff0..80804392c0753 100644 --- a/src/query/sql/src/executor/physical_plans/physical_broadcast.rs +++ b/src/query/sql/src/executor/physical_plans/physical_broadcast.rs @@ -17,7 +17,7 @@ use databend_common_exception::Result; use super::Exchange; use super::FragmentKind; -use crate::executor::PhysicalPlan; +use crate::executor::{IPhysicalPlan, PhysicalPlan}; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct BroadcastSource { @@ -25,6 +25,8 @@ pub struct BroadcastSource { pub broadcast_id: u32, } +impl IPhysicalPlan for BroadcastSource {} + #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct BroadcastSink { pub plan_id: u32, @@ -32,6 +34,10 @@ pub struct BroadcastSink { pub input: Box, } +impl IPhysicalPlan for BroadcastSink { + +} + pub fn build_broadcast_plan(broadcast_id: u32) -> Result { let broadcast_source = Box::new(PhysicalPlan::BroadcastSource(BroadcastSource { plan_id: 0, diff --git a/src/query/sql/src/executor/physical_plans/physical_cache_scan.rs b/src/query/sql/src/executor/physical_plans/physical_cache_scan.rs index 98575dca5b3d5..66e28c83f6082 100644 --- a/src/query/sql/src/executor/physical_plans/physical_cache_scan.rs +++ b/src/query/sql/src/executor/physical_plans/physical_cache_scan.rs @@ -16,7 +16,7 @@ use databend_common_exception::Result; use databend_common_expression::DataSchemaRef; use databend_common_expression::DataSchemaRefExt; -use crate::executor::PhysicalPlan; +use crate::executor::{IPhysicalPlan, PhysicalPlan}; use crate::executor::PhysicalPlanBuilder; use crate::plans::CacheSource; use crate::ColumnSet; @@ -29,6 +29,10 @@ pub struct CacheScan { pub output_schema: DataSchemaRef, } +impl IPhysicalPlan for CacheScan { + +} + impl CacheScan { pub fn output_schema(&self) -> Result { Ok(self.output_schema.clone()) diff --git a/src/query/sql/src/executor/physical_plans/physical_column_mutation.rs b/src/query/sql/src/executor/physical_plans/physical_column_mutation.rs index d3475e796ee16..499a54e8331e3 100644 --- a/src/query/sql/src/executor/physical_plans/physical_column_mutation.rs +++ b/src/query/sql/src/executor/physical_plans/physical_column_mutation.rs @@ -17,7 +17,7 @@ use std::collections::HashMap; use databend_common_expression::RemoteExpr; use databend_common_meta_app::schema::TableInfo; use databend_storages_common_table_meta::meta::TableMetaTimestamps; - +use crate::executor::IPhysicalPlan; use crate::executor::physical_plan::PhysicalPlan; use crate::executor::physical_plans::MutationKind; @@ -34,3 +34,7 @@ pub struct ColumnMutation { pub has_filter_column: bool, pub table_meta_timestamps: TableMetaTimestamps, } + +impl IPhysicalPlan for ColumnMutation { + +} diff --git a/src/query/sql/src/executor/physical_plans/physical_commit_sink.rs b/src/query/sql/src/executor/physical_plans/physical_commit_sink.rs index 81f24be9f22b1..88389186eff83 100644 --- a/src/query/sql/src/executor/physical_plans/physical_commit_sink.rs +++ b/src/query/sql/src/executor/physical_plans/physical_commit_sink.rs @@ -21,7 +21,7 @@ use databend_storages_common_table_meta::meta::TableMetaTimestamps; use databend_storages_common_table_meta::meta::TableSnapshot; use crate::executor::physical_plans::common::MutationKind; -use crate::executor::PhysicalPlan; +use crate::executor::{IPhysicalPlan, PhysicalPlan}; use crate::plans::TruncateMode; // serde is required by `PhysicalPlan` @@ -41,6 +41,10 @@ pub struct CommitSink { pub recluster_info: Option, } +impl IPhysicalPlan for CommitSink { + +} + #[derive(serde::Serialize, serde::Deserialize, Clone, Debug)] pub enum CommitType { Truncate { diff --git a/src/query/sql/src/executor/physical_plans/physical_compact_source.rs b/src/query/sql/src/executor/physical_plans/physical_compact_source.rs index 70bb52b44da9a..b15e24023385e 100644 --- a/src/query/sql/src/executor/physical_plans/physical_compact_source.rs +++ b/src/query/sql/src/executor/physical_plans/physical_compact_source.rs @@ -28,7 +28,7 @@ use crate::executor::physical_plans::CommitType; use crate::executor::physical_plans::Exchange; use crate::executor::physical_plans::FragmentKind; use crate::executor::physical_plans::MutationKind; -use crate::executor::PhysicalPlan; +use crate::executor::{IPhysicalPlan, PhysicalPlan}; use crate::executor::PhysicalPlanBuilder; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] @@ -40,6 +40,10 @@ pub struct CompactSource { pub table_meta_timestamps: TableMetaTimestamps, } +impl IPhysicalPlan for CompactSource { + +} + impl PhysicalPlanBuilder { pub async fn build_compact_block( &mut self, diff --git a/src/query/sql/src/executor/physical_plans/physical_constant_table_scan.rs b/src/query/sql/src/executor/physical_plans/physical_constant_table_scan.rs index a67a9859348b7..c07b15ae08919 100644 --- a/src/query/sql/src/executor/physical_plans/physical_constant_table_scan.rs +++ b/src/query/sql/src/executor/physical_plans/physical_constant_table_scan.rs @@ -16,7 +16,7 @@ use databend_common_exception::Result; use databend_common_expression::Column; use databend_common_expression::DataSchemaRef; -use crate::executor::PhysicalPlan; +use crate::executor::{IPhysicalPlan, PhysicalPlan}; use crate::executor::PhysicalPlanBuilder; use crate::ColumnSet; use crate::IndexType; @@ -30,6 +30,10 @@ pub struct ConstantTableScan { pub output_schema: DataSchemaRef, } +impl IPhysicalPlan for ConstantTableScan { + +} + impl ConstantTableScan { pub fn output_schema(&self) -> Result { Ok(self.output_schema.clone()) diff --git a/src/query/sql/src/executor/physical_plans/physical_copy_into_location.rs b/src/query/sql/src/executor/physical_plans/physical_copy_into_location.rs index 5b069b8587664..691ef13dc1dc4 100644 --- a/src/query/sql/src/executor/physical_plans/physical_copy_into_location.rs +++ b/src/query/sql/src/executor/physical_plans/physical_copy_into_location.rs @@ -21,7 +21,7 @@ use databend_common_expression::DataSchemaRefExt; use databend_common_expression::TableSchemaRef; use databend_storages_common_stage::CopyIntoLocationInfo; -use crate::executor::PhysicalPlan; +use crate::executor::{IPhysicalPlan, PhysicalPlan}; use crate::ColumnBinding; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] @@ -34,6 +34,10 @@ pub struct CopyIntoLocation { pub info: CopyIntoLocationInfo, } +impl IPhysicalPlan for CopyIntoLocation { + +} + impl CopyIntoLocation { pub fn output_schema(&self) -> Result { Ok(DataSchemaRefExt::create(vec![ diff --git a/src/query/sql/src/executor/physical_plans/physical_copy_into_table.rs b/src/query/sql/src/executor/physical_plans/physical_copy_into_table.rs index 6d678ad12efce..f1da86f95bdea 100644 --- a/src/query/sql/src/executor/physical_plans/physical_copy_into_table.rs +++ b/src/query/sql/src/executor/physical_plans/physical_copy_into_table.rs @@ -25,6 +25,7 @@ use crate::executor::physical_plan::PhysicalPlan; use crate::plans::CopyIntoTableMode; use crate::plans::ValidationMode; use crate::ColumnBinding; +use crate::executor::IPhysicalPlan; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct CopyIntoTable { @@ -44,6 +45,10 @@ pub struct CopyIntoTable { pub table_meta_timestamps: TableMetaTimestamps, } +impl IPhysicalPlan for CopyIntoTable { + +} + #[derive(Clone, Debug, serde::Serialize, serde::Deserialize, EnumAsInner)] pub enum CopyIntoTableSource { Query(Box), diff --git a/src/query/sql/src/executor/physical_plans/physical_distributed_insert_select.rs b/src/query/sql/src/executor/physical_plans/physical_distributed_insert_select.rs index 678cf7b210637..049c076bbf88f 100644 --- a/src/query/sql/src/executor/physical_plans/physical_distributed_insert_select.rs +++ b/src/query/sql/src/executor/physical_plans/physical_distributed_insert_select.rs @@ -16,7 +16,7 @@ use databend_common_expression::DataSchemaRef; use databend_common_meta_app::schema::TableInfo; use databend_storages_common_table_meta::meta::TableMetaTimestamps; -use crate::executor::PhysicalPlan; +use crate::executor::{IPhysicalPlan, PhysicalPlan}; use crate::ColumnBinding; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] @@ -32,3 +32,7 @@ pub struct DistributedInsertSelect { pub cast_needed: bool, pub table_meta_timestamps: TableMetaTimestamps, } + +impl IPhysicalPlan for DistributedInsertSelect { + +} diff --git a/src/query/sql/src/executor/physical_plans/physical_eval_scalar.rs b/src/query/sql/src/executor/physical_plans/physical_eval_scalar.rs index 89c30b3bc8e92..5e6f0bb91a9b7 100644 --- a/src/query/sql/src/executor/physical_plans/physical_eval_scalar.rs +++ b/src/query/sql/src/executor/physical_plans/physical_eval_scalar.rs @@ -39,21 +39,65 @@ use crate::plans::ScalarExpr; use crate::plans::ScalarItem; use crate::plans::Visitor; use crate::ColumnSet; +use crate::executor::{IPhysicalPlan, PhysicalPlanMeta}; use crate::IndexType; use crate::TypeCheck; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct EvalScalar { + meta: PhysicalPlanMeta, // A unique id of operator in a `PhysicalPlan` tree, only used for display. pub plan_id: u32, pub projections: ColumnSet, - pub input: Box, + pub input: Box, pub exprs: Vec<(RemoteExpr, IndexType)>, /// Only used for explain pub stat_info: Option, } +impl IPhysicalPlan for EvalScalar { + fn get_meta(&self) -> &PhysicalPlanMeta { + &self.meta + } + + fn get_meta_mut(&mut self) -> &mut PhysicalPlanMeta { + &mut self.meta + } + + fn output_schema(&self) -> Result { + if self.exprs.is_empty() { + return self.input.output_schema(); + } + let input_schema = self.input.output_schema()?; + let mut fields = Vec::with_capacity(self.projections.len()); + for (i, field) in input_schema.fields().iter().enumerate() { + if self.projections.contains(&i) { + fields.push(field.clone()); + } + } + let input_column_nums = input_schema.num_fields(); + for (i, (expr, index)) in self.exprs.iter().enumerate() { + let i = i + input_column_nums; + if !self.projections.contains(&i) { + continue; + } + let name = index.to_string(); + let data_type = expr.as_expr(&BUILTIN_FUNCTIONS).data_type().clone(); + fields.push(DataField::new(&name, data_type)); + } + Ok(DataSchemaRefExt::create(fields)) + } + + fn children<'a>(&'a self) -> Box> + 'a> { + Box::new(std::iter::once(&self.input)) + } + + fn children_mut<'a>(&'a self) -> Box> + 'a> { + Box::new(std::iter::once(&mut self.input)) + } +} + impl EvalScalar { pub fn output_schema(&self) -> Result { if self.exprs.is_empty() { diff --git a/src/query/sql/src/executor/physical_plans/physical_exchange.rs b/src/query/sql/src/executor/physical_plans/physical_exchange.rs index 1e831519c415b..151a6392d6d94 100644 --- a/src/query/sql/src/executor/physical_plans/physical_exchange.rs +++ b/src/query/sql/src/executor/physical_plans/physical_exchange.rs @@ -19,7 +19,7 @@ use databend_common_expression::RemoteExpr; use databend_common_functions::BUILTIN_FUNCTIONS; use crate::executor::physical_plans::common::FragmentKind; -use crate::executor::PhysicalPlan; +use crate::executor::{IPhysicalPlan, PhysicalPlan}; use crate::executor::PhysicalPlanBuilder; use crate::optimizer::ir::SExpr; use crate::ColumnSet; @@ -37,6 +37,10 @@ pub struct Exchange { pub allow_adjust_parallelism: bool, } +impl IPhysicalPlan for Exchange { + +} + impl Exchange { pub fn output_schema(&self) -> Result { self.input.output_schema() diff --git a/src/query/sql/src/executor/physical_plans/physical_exchange_sink.rs b/src/query/sql/src/executor/physical_plans/physical_exchange_sink.rs index cb5ad775bf2bb..1c936bf177962 100644 --- a/src/query/sql/src/executor/physical_plans/physical_exchange_sink.rs +++ b/src/query/sql/src/executor/physical_plans/physical_exchange_sink.rs @@ -17,7 +17,7 @@ use databend_common_expression::DataSchemaRef; use databend_common_expression::RemoteExpr; use crate::executor::physical_plans::common::FragmentKind; -use crate::executor::PhysicalPlan; +use crate::executor::{IPhysicalPlan, PhysicalPlan}; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct ExchangeSink { @@ -38,6 +38,10 @@ pub struct ExchangeSink { pub allow_adjust_parallelism: bool, } +impl IPhysicalPlan for ExchangeSink { + +} + impl ExchangeSink { pub fn output_schema(&self) -> Result { Ok(self.schema.clone()) diff --git a/src/query/sql/src/executor/physical_plans/physical_exchange_source.rs b/src/query/sql/src/executor/physical_plans/physical_exchange_source.rs index 08ec4b44b4f86..fabb0f1f458c5 100644 --- a/src/query/sql/src/executor/physical_plans/physical_exchange_source.rs +++ b/src/query/sql/src/executor/physical_plans/physical_exchange_source.rs @@ -14,6 +14,7 @@ use databend_common_exception::Result; use databend_common_expression::DataSchemaRef; +use crate::executor::IPhysicalPlan; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct ExchangeSource { @@ -28,6 +29,10 @@ pub struct ExchangeSource { pub query_id: String, } +impl IPhysicalPlan for ExchangeSource { + +} + impl ExchangeSource { pub fn output_schema(&self) -> Result { Ok(self.schema.clone()) diff --git a/src/query/sql/src/executor/physical_plans/physical_expression_scan.rs b/src/query/sql/src/executor/physical_plans/physical_expression_scan.rs index 5721f80368321..aabb23af935aa 100644 --- a/src/query/sql/src/executor/physical_plans/physical_expression_scan.rs +++ b/src/query/sql/src/executor/physical_plans/physical_expression_scan.rs @@ -18,7 +18,7 @@ use databend_common_expression::DataSchemaRef; use databend_common_expression::RemoteExpr; use databend_common_functions::BUILTIN_FUNCTIONS; -use crate::executor::PhysicalPlan; +use crate::executor::{IPhysicalPlan, PhysicalPlan}; use crate::executor::PhysicalPlanBuilder; use crate::optimizer::ir::SExpr; use crate::ColumnSet; @@ -33,6 +33,10 @@ pub struct ExpressionScan { pub output_schema: DataSchemaRef, } +impl IPhysicalPlan for ExpressionScan { + +} + impl ExpressionScan { pub fn output_schema(&self) -> Result { Ok(self.output_schema.clone()) diff --git a/src/query/sql/src/executor/physical_plans/physical_filter.rs b/src/query/sql/src/executor/physical_plans/physical_filter.rs index 3051e5ec9307b..8a8ce1fae89be 100644 --- a/src/query/sql/src/executor/physical_plans/physical_filter.rs +++ b/src/query/sql/src/executor/physical_plans/physical_filter.rs @@ -19,7 +19,7 @@ use databend_common_expression::DataSchemaRefExt; use databend_common_expression::RemoteExpr; use databend_common_functions::BUILTIN_FUNCTIONS; -use crate::executor::cast_expr_to_non_null_boolean; +use crate::executor::{cast_expr_to_non_null_boolean, IPhysicalPlan, PhysicalPlanMeta}; use crate::executor::explain::PlanStatsInfo; use crate::executor::PhysicalPlan; use crate::executor::PhysicalPlanBuilder; @@ -29,10 +29,11 @@ use crate::TypeCheck; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct Filter { + meta: PhysicalPlanMeta, // A unique id of operator in a `PhysicalPlan` tree, only used for display. pub plan_id: u32, pub projections: ColumnSet, - pub input: Box, + pub input: Box, // Assumption: expression's data type must be `DataType::Boolean`. pub predicates: Vec, @@ -40,6 +41,35 @@ pub struct Filter { pub stat_info: Option, } +impl IPhysicalPlan for Filter { + fn get_meta(&self) -> &PhysicalPlanMeta { + &self.meta + } + + fn get_meta_mut(&mut self) -> &mut PhysicalPlanMeta { + &mut self.meta + } + + fn children<'a>(&'a self) -> Box> + 'a> { + Box::new(std::iter::once(&self.input)) + } + + fn children_mut<'a>(&'a self) -> Box> + 'a> { + Box::new(std::iter::once(&mut self.input)) + } + + fn output_schema(&self) -> Result { + let input_schema = self.input.output_schema()?; + let mut fields = Vec::with_capacity(self.projections.len()); + for (i, field) in input_schema.fields().iter().enumerate() { + if self.projections.contains(&i) { + fields.push(field.clone()); + } + } + Ok(DataSchemaRefExt::create(fields)) + } +} + impl Filter { pub fn output_schema(&self) -> Result { let input_schema = self.input.output_schema()?; diff --git a/src/query/sql/src/executor/physical_plans/physical_hash_join.rs b/src/query/sql/src/executor/physical_plans/physical_hash_join.rs index 72b233e170f32..dc7004b1e372d 100644 --- a/src/query/sql/src/executor/physical_plans/physical_hash_join.rs +++ b/src/query/sql/src/executor/physical_plans/physical_hash_join.rs @@ -31,7 +31,7 @@ use super::physical_join_filter::PhysicalRuntimeFilters; use super::JoinRuntimeFilter; use crate::executor::explain::PlanStatsInfo; use crate::executor::physical_plans::Exchange; -use crate::executor::PhysicalPlan; +use crate::executor::{IPhysicalPlan, PhysicalPlan, PhysicalPlanMeta}; use crate::executor::PhysicalPlanBuilder; use crate::optimizer::ir::SExpr; use crate::plans::Join; @@ -68,6 +68,7 @@ type MergedFieldsResult = ( pub struct HashJoin { // A unique id of operator in a `PhysicalPlan` tree, only used for display. pub plan_id: u32, + meta: PhysicalPlanMeta, // After building the probe key and build key, we apply probe_projections to probe_datablock // and build_projections to build_datablock, which can help us reduce memory usage and calls // of expensive functions (take_compacted_indices and gather), after processing other_conditions, @@ -108,6 +109,24 @@ pub struct HashJoin { pub broadcast_id: Option, } +impl IPhysicalPlan for HashJoin { + fn get_meta(&self) -> &PhysicalPlanMeta { + &self.meta + } + + fn get_meta_mut(&mut self) -> &mut PhysicalPlanMeta { + &mut self.meta + } + + fn output_schema(&self) -> Result { + Ok(self.output_schema.clone()) + } + + fn children<'a>(&'a self) -> Box> + 'a> { + todo!() + } +} + impl HashJoin { pub fn output_schema(&self) -> Result { Ok(self.output_schema.clone()) @@ -225,11 +244,11 @@ impl PhysicalPlanBuilder { // Unify the data types of the left and right exchange keys if let ( PhysicalPlan::Exchange(Exchange { - keys: probe_keys, .. - }), + keys: probe_keys, .. + }), PhysicalPlan::Exchange(Exchange { - keys: build_keys, .. - }), + keys: build_keys, .. + }), ) = (probe_side.as_mut(), build_side.as_mut()) { let cast_rules = &BUILTIN_FUNCTIONS.get_auto_cast_rules("eq"); @@ -241,12 +260,12 @@ impl PhysicalPlanBuilder { build_expr.data_type().clone(), cast_rules, ) - .ok_or_else(|| { - ErrorCode::IllegalDataType(format!( - "Cannot find common type for probe key {:?} and build key {:?}", - &probe_expr, &build_expr - )) - })?; + .ok_or_else(|| { + ErrorCode::IllegalDataType(format!( + "Cannot find common type for probe key {:?} and build key {:?}", + &probe_expr, &build_expr + )) + })?; *probe_key = check_cast( probe_expr.span(), false, @@ -254,7 +273,7 @@ impl PhysicalPlanBuilder { &common_ty, &BUILTIN_FUNCTIONS, )? - .as_remote_expr(); + .as_remote_expr(); *build_key = check_cast( build_expr.span(), false, @@ -262,7 +281,7 @@ impl PhysicalPlanBuilder { &common_ty, &BUILTIN_FUNCTIONS, )? - .as_remote_expr(); + .as_remote_expr(); } } @@ -351,9 +370,9 @@ impl PhysicalPlanBuilder { .data_type() .remove_nullable() == build_schema - .field(build_index) - .data_type() - .remove_nullable() + .field(build_index) + .data_type() + .remove_nullable() { probe_to_build_index.push(((probe_index, false), build_index)); if !pre_column_projections.contains(&left.column.index) { @@ -956,6 +975,6 @@ impl PhysicalPlanBuilder { build_keys, probe_keys, ) - .await + .await } } diff --git a/src/query/sql/src/executor/physical_plans/physical_limit.rs b/src/query/sql/src/executor/physical_plans/physical_limit.rs index b0c3e64ecd713..ea0f523f66fc1 100644 --- a/src/query/sql/src/executor/physical_plans/physical_limit.rs +++ b/src/query/sql/src/executor/physical_plans/physical_limit.rs @@ -20,7 +20,7 @@ use databend_common_expression::ROW_ID_COL_NAME; use crate::executor::explain::PlanStatsInfo; use crate::executor::physical_plans::physical_row_fetch::RowFetch; -use crate::executor::PhysicalPlan; +use crate::executor::{IPhysicalPlan, PhysicalPlan, PhysicalPlanMeta}; use crate::executor::PhysicalPlanBuilder; use crate::optimizer::ir::SExpr; use crate::ColumnEntry; @@ -30,7 +30,8 @@ use crate::ColumnSet; pub struct Limit { // A unique id of operator in a `PhysicalPlan` tree, only used for display. pub plan_id: u32, - pub input: Box, + meta: PhysicalPlanMeta, + pub input: Box, pub limit: Option, pub offset: usize, @@ -38,6 +39,24 @@ pub struct Limit { pub stat_info: Option, } +impl IPhysicalPlan for Limit { + fn get_meta(&self) -> &PhysicalPlanMeta { + &self.meta + } + + fn get_meta_mut(&mut self) -> &mut PhysicalPlanMeta { + &mut self.meta + } + + fn children<'a>(&'a self) -> Box> + 'a> { + Box::new(std::iter::once(&self.input)) + } + + fn children_mut<'a>(&'a self) -> Box> + 'a> { + Box::new(std::iter::once(&mut self.input)) + } +} + impl Limit { pub fn output_schema(&self) -> Result { self.input.output_schema() diff --git a/src/query/sql/src/executor/physical_plans/physical_multi_table_insert.rs b/src/query/sql/src/executor/physical_plans/physical_multi_table_insert.rs index 41c1f88954dd6..8f4a08640643a 100644 --- a/src/query/sql/src/executor/physical_plans/physical_multi_table_insert.rs +++ b/src/query/sql/src/executor/physical_plans/physical_multi_table_insert.rs @@ -23,7 +23,7 @@ use databend_common_meta_app::schema::TableInfo; use databend_common_meta_app::schema::UpdateStreamMetaReq; use databend_storages_common_table_meta::meta::TableMetaTimestamps; -use crate::executor::PhysicalPlan; +use crate::executor::{IPhysicalPlan, PhysicalPlan}; use crate::ColumnSet; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct Duplicate { @@ -32,6 +32,8 @@ pub struct Duplicate { pub n: usize, } +impl IPhysicalPlan for Duplicate {} + #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct Shuffle { pub plan_id: u32, @@ -39,6 +41,8 @@ pub struct Shuffle { pub strategy: ShuffleStrategy, } +impl IPhysicalPlan for Shuffle {} + #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub enum ShuffleStrategy { Transpose(usize), @@ -73,6 +77,8 @@ pub struct ChunkFilter { pub predicates: Vec>, } +impl IPhysicalPlan for ChunkFilter {} + #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct ChunkEvalScalar { pub plan_id: u32, @@ -80,6 +86,8 @@ pub struct ChunkEvalScalar { pub eval_scalars: Vec>, } +impl IPhysicalPlan for ChunkEvalScalar {} + #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct MultiInsertEvalScalar { pub remote_exprs: Vec, @@ -93,6 +101,8 @@ pub struct ChunkCastSchema { pub cast_schemas: Vec>, } +impl IPhysicalPlan for ChunkCastSchema {} + #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct CastSchema { pub source_schema: DataSchemaRef, @@ -106,6 +116,8 @@ pub struct ChunkFillAndReorder { pub fill_and_reorders: Vec>, } +impl IPhysicalPlan for ChunkFillAndReorder {} + #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct FillAndReorder { pub source_schema: DataSchemaRef, @@ -119,6 +131,8 @@ pub struct ChunkAppendData { pub target_tables: Vec, } +impl IPhysicalPlan for ChunkAppendData {} + #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct SerializableTable { pub target_catalog_info: Arc, @@ -133,6 +147,8 @@ pub struct ChunkMerge { pub group_ids: Vec, } +impl IPhysicalPlan for ChunkMerge {} + #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct ChunkCommitInsert { pub plan_id: u32, @@ -142,3 +158,7 @@ pub struct ChunkCommitInsert { pub deduplicated_label: Option, pub targets: Vec, } + +impl IPhysicalPlan for ChunkCommitInsert { + +} diff --git a/src/query/sql/src/executor/physical_plans/physical_mutation.rs b/src/query/sql/src/executor/physical_plans/physical_mutation.rs index 9feb40c125a1d..78deb7450b46b 100644 --- a/src/query/sql/src/executor/physical_plans/physical_mutation.rs +++ b/src/query/sql/src/executor/physical_plans/physical_mutation.rs @@ -57,7 +57,7 @@ use crate::executor::physical_plans::MutationManipulate; use crate::executor::physical_plans::MutationOrganize; use crate::executor::physical_plans::MutationSplit; use crate::executor::physical_plans::RowFetch; -use crate::executor::PhysicalPlanBuilder; +use crate::executor::{IPhysicalPlan, PhysicalPlanBuilder}; use crate::optimizer::ir::SExpr; use crate::parse_computed_expr; use crate::plans::BoundColumnRef; @@ -95,6 +95,10 @@ pub struct Mutation { pub table_meta_timestamps: TableMetaTimestamps, } +impl IPhysicalPlan for Mutation { + +} + impl PhysicalPlanBuilder { pub async fn build_mutation( &mut self, diff --git a/src/query/sql/src/executor/physical_plans/physical_mutation_into_organize.rs b/src/query/sql/src/executor/physical_plans/physical_mutation_into_organize.rs index f2d6df77da87c..62c555473b0ae 100644 --- a/src/query/sql/src/executor/physical_plans/physical_mutation_into_organize.rs +++ b/src/query/sql/src/executor/physical_plans/physical_mutation_into_organize.rs @@ -16,6 +16,7 @@ use databend_common_exception::Result; use databend_common_expression::DataSchemaRef; use crate::binder::MutationStrategy; +use crate::executor::IPhysicalPlan; use crate::executor::physical_plan::PhysicalPlan; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] @@ -25,6 +26,10 @@ pub struct MutationOrganize { pub strategy: MutationStrategy, } +impl IPhysicalPlan for MutationOrganize { + +} + impl MutationOrganize { pub fn output_schema(&self) -> Result { self.input.output_schema() diff --git a/src/query/sql/src/executor/physical_plans/physical_mutation_into_split.rs b/src/query/sql/src/executor/physical_plans/physical_mutation_into_split.rs index f94b1955afddb..ace7afb551215 100644 --- a/src/query/sql/src/executor/physical_plans/physical_mutation_into_split.rs +++ b/src/query/sql/src/executor/physical_plans/physical_mutation_into_split.rs @@ -14,7 +14,7 @@ use databend_common_exception::Result; use databend_common_expression::DataSchemaRef; - +use crate::executor::IPhysicalPlan; use crate::executor::physical_plan::PhysicalPlan; use crate::IndexType; @@ -25,6 +25,10 @@ pub struct MutationSplit { pub split_index: IndexType, } +impl IPhysicalPlan for MutationSplit { + +} + impl MutationSplit { pub fn output_schema(&self) -> Result { self.input.output_schema() diff --git a/src/query/sql/src/executor/physical_plans/physical_mutation_manipulate.rs b/src/query/sql/src/executor/physical_plans/physical_mutation_manipulate.rs index 50976f11327be..38b21206c90e9 100644 --- a/src/query/sql/src/executor/physical_plans/physical_mutation_manipulate.rs +++ b/src/query/sql/src/executor/physical_plans/physical_mutation_manipulate.rs @@ -21,6 +21,7 @@ use databend_common_expression::RemoteExpr; use databend_common_meta_app::schema::TableInfo; use crate::binder::MutationStrategy; +use crate::executor::IPhysicalPlan; use crate::executor::physical_plan::PhysicalPlan; pub type MatchExpr = Vec<(Option, Option>)>; @@ -43,6 +44,10 @@ pub struct MutationManipulate { pub unmatched_schema: DataSchemaRef, } +impl IPhysicalPlan for MutationManipulate { + +} + impl MutationManipulate { pub fn output_schema(&self) -> Result { self.input.output_schema() diff --git a/src/query/sql/src/executor/physical_plans/physical_mutation_source.rs b/src/query/sql/src/executor/physical_plans/physical_mutation_source.rs index b04c294220ab5..e7ebae33d3c19 100644 --- a/src/query/sql/src/executor/physical_plans/physical_mutation_source.rs +++ b/src/query/sql/src/executor/physical_plans/physical_mutation_source.rs @@ -27,7 +27,7 @@ use databend_common_functions::BUILTIN_FUNCTIONS; use databend_common_meta_app::schema::TableInfo; use crate::binder::MutationType; -use crate::executor::cast_expr_to_non_null_boolean; +use crate::executor::{cast_expr_to_non_null_boolean, IPhysicalPlan}; use crate::executor::PhysicalPlan; use crate::executor::PhysicalPlanBuilder; use crate::ColumnSet; @@ -50,6 +50,10 @@ pub struct MutationSource { pub statistics: PartStatistics, } +impl IPhysicalPlan for MutationSource { + +} + impl MutationSource { pub fn output_schema(&self) -> Result { Ok(self.output_schema.clone()) diff --git a/src/query/sql/src/executor/physical_plans/physical_project_set.rs b/src/query/sql/src/executor/physical_plans/physical_project_set.rs index 99b85519223b8..740341d319fad 100644 --- a/src/query/sql/src/executor/physical_plans/physical_project_set.rs +++ b/src/query/sql/src/executor/physical_plans/physical_project_set.rs @@ -21,7 +21,7 @@ use databend_common_expression::RemoteExpr; use databend_common_functions::BUILTIN_FUNCTIONS; use crate::executor::explain::PlanStatsInfo; -use crate::executor::PhysicalPlan; +use crate::executor::{IPhysicalPlan, PhysicalPlan, PhysicalPlanMeta}; use crate::executor::PhysicalPlanBuilder; use crate::optimizer::ir::SExpr; use crate::ColumnSet; @@ -32,14 +32,50 @@ use crate::TypeCheck; pub struct ProjectSet { // A unique id of operator in a `PhysicalPlan` tree, only used for display. pub plan_id: u32, + meta: PhysicalPlanMeta, pub projections: ColumnSet, - pub input: Box, + pub input: Box, pub srf_exprs: Vec<(RemoteExpr, IndexType)>, // Only used for explain pub stat_info: Option, } +impl IPhysicalPlan for ProjectSet { + fn get_meta(&self) -> &PhysicalPlanMeta { + &self.meta + } + + fn get_meta_mut(&mut self) -> &mut PhysicalPlanMeta { + &mut self.meta + } + + fn output_schema(&self) -> Result { + let input_schema = self.input.output_schema()?; + let mut fields = Vec::with_capacity(input_schema.num_fields() + self.srf_exprs.len()); + for (i, field) in input_schema.fields().iter().enumerate() { + if self.projections.contains(&i) { + fields.push(field.clone()); + } + } + fields.extend(self.srf_exprs.iter().map(|(srf, index)| { + DataField::new( + &index.to_string(), + srf.as_expr(&BUILTIN_FUNCTIONS).data_type().clone(), + ) + })); + Ok(DataSchemaRefExt::create(fields)) + } + + fn children<'a>(&'a self) -> Box> + 'a> { + Box::new(std::iter::once(&self.input)) + } + + fn children_mut<'a>(&'a self) -> Box> + 'a> { + Box::new(std::iter::once(&mut self.input)) + } +} + impl ProjectSet { pub fn output_schema(&self) -> Result { let input_schema = self.input.output_schema()?; diff --git a/src/query/sql/src/executor/physical_plans/physical_r_cte_scan.rs b/src/query/sql/src/executor/physical_plans/physical_r_cte_scan.rs index 459a6c3c91bd2..48a16c37e7679 100644 --- a/src/query/sql/src/executor/physical_plans/physical_r_cte_scan.rs +++ b/src/query/sql/src/executor/physical_plans/physical_r_cte_scan.rs @@ -19,7 +19,7 @@ use databend_common_expression::DataSchemaRef; use databend_common_expression::DataSchemaRefExt; use crate::executor::explain::PlanStatsInfo; -use crate::executor::PhysicalPlan; +use crate::executor::{IPhysicalPlan, PhysicalPlan}; use crate::executor::PhysicalPlanBuilder; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] @@ -31,6 +31,10 @@ pub struct RecursiveCteScan { pub stat: PlanStatsInfo, } +impl IPhysicalPlan for RecursiveCteScan { + +} + impl RecursiveCteScan { pub fn output_schema(&self) -> Result { Ok(self.output_schema.clone()) diff --git a/src/query/sql/src/executor/physical_plans/physical_range_join.rs b/src/query/sql/src/executor/physical_plans/physical_range_join.rs index 94317540f2f99..b6ca2e654ae84 100644 --- a/src/query/sql/src/executor/physical_plans/physical_range_join.rs +++ b/src/query/sql/src/executor/physical_plans/physical_range_join.rs @@ -26,7 +26,7 @@ use databend_common_functions::BUILTIN_FUNCTIONS; use crate::binder::wrap_cast; use crate::binder::JoinPredicate; use crate::executor::explain::PlanStatsInfo; -use crate::executor::PhysicalPlan; +use crate::executor::{IPhysicalPlan, PhysicalPlan}; use crate::executor::PhysicalPlanBuilder; use crate::optimizer::ir::RelExpr; use crate::optimizer::ir::RelationalProperty; @@ -57,6 +57,10 @@ pub struct RangeJoin { pub stat_info: Option, } +impl IPhysicalPlan for RangeJoin { + +} + impl RangeJoin { pub fn output_schema(&self) -> Result { Ok(self.output_schema.clone()) diff --git a/src/query/sql/src/executor/physical_plans/physical_recluster.rs b/src/query/sql/src/executor/physical_plans/physical_recluster.rs index 9227c86b64199..592bc74a142c6 100644 --- a/src/query/sql/src/executor/physical_plans/physical_recluster.rs +++ b/src/query/sql/src/executor/physical_plans/physical_recluster.rs @@ -16,7 +16,7 @@ use databend_common_catalog::plan::ReclusterTask; use databend_common_meta_app::schema::TableInfo; use databend_storages_common_table_meta::meta::TableMetaTimestamps; -use crate::executor::PhysicalPlan; +use crate::executor::{IPhysicalPlan, PhysicalPlan}; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct Recluster { @@ -26,6 +26,10 @@ pub struct Recluster { pub table_meta_timestamps: TableMetaTimestamps, } +impl IPhysicalPlan for Recluster { + +} + #[derive(serde::Serialize, serde::Deserialize, Clone, Debug)] pub struct HilbertPartition { pub plan_id: u32, @@ -35,3 +39,7 @@ pub struct HilbertPartition { pub table_meta_timestamps: TableMetaTimestamps, pub rows_per_block: usize, } + +impl IPhysicalPlan for HilbertPartition { + +} diff --git a/src/query/sql/src/executor/physical_plans/physical_replace_async_source.rs b/src/query/sql/src/executor/physical_plans/physical_replace_async_source.rs index f7a640fd9cc00..872896518ec96 100644 --- a/src/query/sql/src/executor/physical_plans/physical_replace_async_source.rs +++ b/src/query/sql/src/executor/physical_plans/physical_replace_async_source.rs @@ -13,7 +13,7 @@ // limitations under the License. use databend_common_expression::DataSchemaRef; - +use crate::executor::IPhysicalPlan; use crate::plans::InsertValue; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] @@ -22,3 +22,7 @@ pub struct ReplaceAsyncSourcer { pub schema: DataSchemaRef, pub source: InsertValue, } + +impl IPhysicalPlan for ReplaceAsyncSourcer { + +} diff --git a/src/query/sql/src/executor/physical_plans/physical_replace_deduplicate.rs b/src/query/sql/src/executor/physical_plans/physical_replace_deduplicate.rs index 24224053d03b4..3f7c5617d7cf4 100644 --- a/src/query/sql/src/executor/physical_plans/physical_replace_deduplicate.rs +++ b/src/query/sql/src/executor/physical_plans/physical_replace_deduplicate.rs @@ -23,7 +23,7 @@ use databend_common_meta_app::schema::TableInfo; use databend_storages_common_table_meta::meta::ColumnStatistics; use crate::executor::physical_plans::common::OnConflictField; -use crate::executor::PhysicalPlan; +use crate::executor::{IPhysicalPlan, PhysicalPlan}; use crate::ColumnBinding; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] @@ -41,6 +41,10 @@ pub struct ReplaceDeduplicate { pub delete_when: Option<(RemoteExpr, String)>, } +impl IPhysicalPlan for ReplaceDeduplicate { + +} + #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct ReplaceSelectCtx { pub select_column_bindings: Vec, diff --git a/src/query/sql/src/executor/physical_plans/physical_replace_into.rs b/src/query/sql/src/executor/physical_plans/physical_replace_into.rs index 75d0af7ac2cd3..e58c4cd97fd6b 100644 --- a/src/query/sql/src/executor/physical_plans/physical_replace_into.rs +++ b/src/query/sql/src/executor/physical_plans/physical_replace_into.rs @@ -20,7 +20,7 @@ use databend_storages_common_table_meta::meta::Location; use databend_storages_common_table_meta::meta::TableMetaTimestamps; use crate::executor::physical_plans::common::OnConflictField; -use crate::executor::PhysicalPlan; +use crate::executor::{IPhysicalPlan, PhysicalPlan}; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct ReplaceInto { @@ -37,3 +37,7 @@ pub struct ReplaceInto { pub need_insert: bool, pub table_meta_timestamps: TableMetaTimestamps, } + +impl IPhysicalPlan for ReplaceInto { + +} diff --git a/src/query/sql/src/executor/physical_plans/physical_row_fetch.rs b/src/query/sql/src/executor/physical_plans/physical_row_fetch.rs index 3a5aca4f7a309..b82415a7b34bc 100644 --- a/src/query/sql/src/executor/physical_plans/physical_row_fetch.rs +++ b/src/query/sql/src/executor/physical_plans/physical_row_fetch.rs @@ -20,13 +20,14 @@ use databend_common_expression::DataSchemaRef; use databend_common_expression::DataSchemaRefExt; use crate::executor::explain::PlanStatsInfo; -use crate::executor::PhysicalPlan; +use crate::executor::{IPhysicalPlan, PhysicalPlan, PhysicalPlanMeta}; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct RowFetch { // A unique id of operator in a `PhysicalPlan` tree, only used for display. pub plan_id: u32, - pub input: Box, + meta: PhysicalPlanMeta, + pub input: Box, // cloned from `input`. pub source: Box, // projection on the source table schema. @@ -39,6 +40,30 @@ pub struct RowFetch { pub stat_info: Option, } +impl IPhysicalPlan for RowFetch { + fn get_meta(&self) -> &PhysicalPlanMeta { + &self.meta + } + + fn get_meta_mut(&mut self) -> &mut PhysicalPlanMeta { + &mut self.meta + } + + fn output_schema(&self) -> Result { + let mut fields = self.input.output_schema()?.fields().clone(); + fields.extend_from_slice(&self.fetched_fields); + Ok(DataSchemaRefExt::create(fields)) + } + + fn children<'a>(&'a self) -> Box> + 'a> { + Box::new(std::iter::once(&self.input)) + } + + fn children_mut<'a>(&'a self) -> Box> + 'a> { + Box::new(std::iter::once(&mut self.input)) + } +} + impl RowFetch { pub fn output_schema(&self) -> Result { let mut fields = self.input.output_schema()?.fields().clone(); diff --git a/src/query/sql/src/executor/physical_plans/physical_sort.rs b/src/query/sql/src/executor/physical_plans/physical_sort.rs index 94b765107f310..915c1c38b7046 100644 --- a/src/query/sql/src/executor/physical_plans/physical_sort.rs +++ b/src/query/sql/src/executor/physical_plans/physical_sort.rs @@ -25,7 +25,7 @@ use crate::executor::physical_plans::common::SortDesc; use crate::executor::physical_plans::WindowPartition; use crate::executor::physical_plans::WindowPartitionTopN; use crate::executor::physical_plans::WindowPartitionTopNFunc; -use crate::executor::PhysicalPlan; +use crate::executor::{IPhysicalPlan, PhysicalPlan, PhysicalPlanMeta}; use crate::executor::PhysicalPlanBuilder; use crate::optimizer::ir::SExpr; use crate::plans::WindowFuncType; @@ -36,7 +36,8 @@ use crate::IndexType; pub struct Sort { /// A unique id of operator in a `PhysicalPlan` tree, only used for display. pub plan_id: u32, - pub input: Box, + meta: PhysicalPlanMeta, + pub input: Box, pub order_by: Vec, /// limit = Limit.limit + Limit.offset pub limit: Option, @@ -49,6 +50,62 @@ pub struct Sort { pub stat_info: Option, } +impl IPhysicalPlan for Sort { + fn get_meta(&self) -> &PhysicalPlanMeta { + &self.meta + } + + fn get_meta_mut(&mut self) -> &mut PhysicalPlanMeta { + &mut self.meta + } + + fn output_schema(&self) -> Result { + let input_schema = self.input.output_schema()?; + let mut fields = input_schema.fields().clone(); + if matches!(self.after_exchange, Some(true)) { + // If the plan is after exchange plan in cluster mode, + // the order column is at the last of the input schema. + debug_assert_eq!(fields.last().unwrap().name(), ORDER_COL_NAME); + debug_assert_eq!( + fields.last().unwrap().data_type(), + &self.order_col_type(&input_schema)? + ); + fields.pop(); + } else { + if let Some(proj) = &self.pre_projection { + let fileted_fields = proj + .iter() + .filter_map(|index| input_schema.field_with_name(&index.to_string()).ok()) + .cloned() + .collect::>(); + if fileted_fields.len() < fields.len() { + // Only if the projection is not a full projection, we need to add a projection transform. + fields = fileted_fields + } + } + + if matches!(self.after_exchange, Some(false)) { + // If the plan is before exchange plan in cluster mode, + // the order column should be added to the output schema. + fields.push(DataField::new( + ORDER_COL_NAME, + self.order_col_type(&input_schema)?, + )); + } + } + + Ok(DataSchemaRefExt::create(fields)) + } + + fn children<'a>(&'a self) -> Box> + 'a> { + Box::new(std::iter::once(&self.input)) + } + + fn children_mut<'a>(&'a self) -> Box> + 'a> { + Box::new(std::iter::once(&mut self.input)) + } +} + impl Sort { fn order_col_type(&self, schema: &DataSchema) -> Result { if self.order_by.len() == 1 { diff --git a/src/query/sql/src/executor/physical_plans/physical_table_scan.rs b/src/query/sql/src/executor/physical_plans/physical_table_scan.rs index 0541b88c9abd1..16067b8b1e9b1 100644 --- a/src/query/sql/src/executor/physical_plans/physical_table_scan.rs +++ b/src/query/sql/src/executor/physical_plans/physical_table_scan.rs @@ -13,14 +13,14 @@ // limitations under the License. use std::collections::btree_map::Entry; -use std::collections::BTreeMap; +use std::collections::{BTreeMap, HashMap}; use std::collections::HashSet; use std::sync::Arc; use databend_common_ast::parser::token::TokenKind; use databend_common_ast::parser::tokenize_sql; use databend_common_catalog::catalog::CatalogManager; -use databend_common_catalog::plan::DataSourcePlan; +use databend_common_catalog::plan::{DataSourcePlan, PartStatistics, PartitionsShuffleKind}; use databend_common_catalog::plan::Filters; use databend_common_catalog::plan::InternalColumn; use databend_common_catalog::plan::PrewhereInfo; @@ -51,7 +51,7 @@ use rand::distributions::Distribution; use rand::thread_rng; use crate::binder::INTERNAL_COLUMN_FACTORY; -use crate::executor::cast_expr_to_non_null_boolean; +use crate::executor::{cast_expr_to_non_null_boolean, IPhysicalPlan, PhysicalPlanMeta}; use crate::executor::explain::PlanStatsInfo; use crate::executor::physical_plans::AddStreamColumn; use crate::executor::table_read_plan::ToReadDataSourcePlan; @@ -73,6 +73,7 @@ use crate::DUMMY_TABLE_INDEX; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct TableScan { + meta: PhysicalPlanMeta, // A unique id of operator in a `PhysicalPlan` tree, only used for display. pub plan_id: u32, pub scan_id: usize, @@ -84,6 +85,30 @@ pub struct TableScan { pub stat_info: Option, } +impl IPhysicalPlan for TableScan { + fn get_meta(&self) -> &PhysicalPlanMeta { + &self.meta + } + + fn get_meta_mut(&mut self) -> &mut PhysicalPlanMeta { + &mut self.meta + } + + fn get_all_data_source(&self, sources: &mut Vec<(u32, Box)>) { + sources.push((self.get_id(), self.source.clone())); + } + + fn set_pruning_stats(&mut self, stats: &mut HashMap) { + if let Some(stat) = stats.remove(&self.get_id()) { + self.source.statistics = stat; + } + } + + fn is_warehouse_distributed_plan(&self) -> bool { + self.source.parts.kind == PartitionsShuffleKind::BroadcastWarehouse + } +} + impl TableScan { pub fn output_fields( schema: TableSchemaRef, @@ -187,8 +212,8 @@ impl PhysicalPlanBuilder { } } ColumnEntry::InternalColumn(TableInternalColumn { - internal_column, .. - }) => { + internal_column, .. + }) => { project_internal_columns.insert(*index, internal_column.to_owned()); } ColumnEntry::VirtualColumn(virtual_column) => { @@ -510,14 +535,14 @@ impl PhysicalPlanBuilder { let column = metadata.column(item.index); let (name, data_type) = match column { ColumnEntry::BaseTableColumn(BaseTableColumn { - column_name, - data_type, - .. - }) => (column_name.clone(), DataType::from(data_type)), + column_name, + data_type, + .. + }) => (column_name.clone(), DataType::from(data_type)), ColumnEntry::InternalColumn(TableInternalColumn { - internal_column, - .. - }) => ( + internal_column, + .. + }) => ( internal_column.column_name().to_owned(), internal_column.data_type(), ), @@ -689,7 +714,7 @@ impl PhysicalPlanBuilder { cast_expr_to_non_null_boolean(pred.as_expr()?.project_column_ref(|col| { output_schema.index_of(&col.index.to_string()).unwrap() }))? - .as_remote_expr(), + .as_remote_expr(), ) }) .transpose()?; @@ -727,7 +752,7 @@ impl PhysicalPlanBuilder { pub fn build_projection<'a>( metadata: &Metadata, schema: &TableSchema, - columns: impl Iterator, + columns: impl Iterator, has_inner_column: bool, ignore_internal_column: bool, add_virtual_source_column: bool, @@ -746,16 +771,16 @@ impl PhysicalPlanBuilder { } ColumnEntry::DerivedColumn(DerivedColumn { alias, .. }) => alias, ColumnEntry::InternalColumn(TableInternalColumn { - internal_column, .. - }) => { + internal_column, .. + }) => { if ignore_internal_column { continue; } internal_column.column_name() } ColumnEntry::VirtualColumn(VirtualColumn { - source_column_name, .. - }) => { + source_column_name, .. + }) => { if add_virtual_source_column { virtual_col_indices .insert(schema.index_of(source_column_name).unwrap()); @@ -783,10 +808,10 @@ impl PhysicalPlanBuilder { let column = metadata.column(*index); match column { ColumnEntry::BaseTableColumn(BaseTableColumn { - column_name, - path_indices, - .. - }) => match path_indices { + column_name, + path_indices, + .. + }) => match path_indices { Some(path_indices) => { col_indices.insert(column.index(), path_indices.to_vec()); } @@ -805,8 +830,8 @@ impl PhysicalPlanBuilder { } } ColumnEntry::VirtualColumn(VirtualColumn { - source_column_name, .. - }) => { + source_column_name, .. + }) => { if add_virtual_source_column { let idx = schema.index_of(source_column_name).unwrap(); col_indices.insert(idx, vec![idx]); diff --git a/src/query/sql/src/executor/physical_plans/physical_udf.rs b/src/query/sql/src/executor/physical_plans/physical_udf.rs index acfd628b36af3..02859a8091cb4 100644 --- a/src/query/sql/src/executor/physical_plans/physical_udf.rs +++ b/src/query/sql/src/executor/physical_plans/physical_udf.rs @@ -23,7 +23,7 @@ use databend_common_expression::DataSchemaRefExt; use databend_common_functions::BUILTIN_FUNCTIONS; use crate::executor::explain::PlanStatsInfo; -use crate::executor::PhysicalPlan; +use crate::executor::{IPhysicalPlan, PhysicalPlan}; use crate::executor::PhysicalPlanBuilder; use crate::optimizer::ir::SExpr; use crate::plans::UDFType; @@ -42,6 +42,10 @@ pub struct Udf { pub stat_info: Option, } +impl IPhysicalPlan for Udf { + +} + impl Udf { pub fn output_schema(&self) -> Result { let input_schema = self.input.output_schema()?; diff --git a/src/query/sql/src/executor/physical_plans/physical_union_all.rs b/src/query/sql/src/executor/physical_plans/physical_union_all.rs index 26ea91d34f81f..4d365dff18cc2 100644 --- a/src/query/sql/src/executor/physical_plans/physical_union_all.rs +++ b/src/query/sql/src/executor/physical_plans/physical_union_all.rs @@ -20,7 +20,7 @@ use databend_common_expression::DataSchemaRefExt; use databend_common_expression::RemoteExpr; use crate::executor::explain::PlanStatsInfo; -use crate::executor::PhysicalPlan; +use crate::executor::{IPhysicalPlan, PhysicalPlan}; use crate::executor::PhysicalPlanBuilder; use crate::optimizer::ir::SExpr; use crate::ColumnSet; @@ -43,6 +43,10 @@ pub struct UnionAll { pub stat_info: Option, } +impl IPhysicalPlan for UnionAll { + +} + impl UnionAll { pub fn output_schema(&self) -> Result { Ok(self.schema.clone()) diff --git a/src/query/sql/src/executor/physical_plans/physical_window.rs b/src/query/sql/src/executor/physical_plans/physical_window.rs index f2cac19cb0fa1..2cba6f8c10fb2 100644 --- a/src/query/sql/src/executor/physical_plans/physical_window.rs +++ b/src/query/sql/src/executor/physical_plans/physical_window.rs @@ -35,7 +35,7 @@ use crate::executor::explain::PlanStatsInfo; use crate::executor::physical_plans::common::AggregateFunctionDesc; use crate::executor::physical_plans::common::AggregateFunctionSignature; use crate::executor::physical_plans::common::SortDesc; -use crate::executor::PhysicalPlan; +use crate::executor::{IPhysicalPlan, PhysicalPlan, PhysicalPlanMeta}; use crate::executor::PhysicalPlanBuilder; use crate::optimizer::ir::SExpr; use crate::plans::WindowFuncFrame; @@ -50,8 +50,9 @@ use crate::TypeCheck; pub struct Window { // A unique id of operator in a `PhysicalPlan` tree, only used for display. pub plan_id: u32, + meta: PhysicalPlanMeta, pub index: IndexType, - pub input: Box, + pub input: Box, pub func: WindowFunction, pub partition_by: Vec, pub order_by: Vec, @@ -59,6 +60,35 @@ pub struct Window { pub limit: Option, } +impl IPhysicalPlan for Window { + fn get_meta(&self) -> &PhysicalPlanMeta { + &self.meta + } + + fn get_meta_mut(&mut self) -> &mut PhysicalPlanMeta { + &mut self.meta + } + + fn output_schema(&self) -> Result { + let input_schema = self.input.output_schema()?; + let mut fields = Vec::with_capacity(input_schema.fields().len() + 1); + fields.extend_from_slice(input_schema.fields()); + fields.push(DataField::new( + &self.index.to_string(), + self.func.data_type()?, + )); + Ok(DataSchemaRefExt::create(fields)) + } + + fn children<'a>(&'a self) -> Box> + 'a> { + Box::new(std::iter::once(&self.input)) + } + + fn children_mut<'a>(&'a self) -> Box> + 'a> { + Box::new(std::iter::once(&mut self.input)) + } +} + impl Window { pub fn output_schema(&self) -> Result { let input_schema = self.input.output_schema()?; diff --git a/src/query/sql/src/executor/physical_plans/physical_window_partition.rs b/src/query/sql/src/executor/physical_plans/physical_window_partition.rs index b0ff12d3f8685..610e43d0c4064 100644 --- a/src/query/sql/src/executor/physical_plans/physical_window_partition.rs +++ b/src/query/sql/src/executor/physical_plans/physical_window_partition.rs @@ -17,13 +17,14 @@ use databend_common_expression::DataSchemaRef; use crate::executor::explain::PlanStatsInfo; use crate::executor::physical_plans::SortDesc; -use crate::executor::PhysicalPlan; +use crate::executor::{IPhysicalPlan, PhysicalPlan, PhysicalPlanMeta}; use crate::IndexType; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct WindowPartition { pub plan_id: u32, - pub input: Box, + meta: PhysicalPlanMeta, + pub input: Box, pub partition_by: Vec, pub order_by: Vec, pub after_exchange: Option, @@ -32,6 +33,24 @@ pub struct WindowPartition { pub stat_info: Option, } +impl IPhysicalPlan for WindowPartition { + fn get_meta(&self) -> &PhysicalPlanMeta { + &self.meta + } + + fn get_meta_mut(&mut self) -> &mut PhysicalPlanMeta { + &mut self.meta + } + + fn children<'a>(&'a self) -> Box> + 'a> { + Box::new(std::iter::once(&self.input)) + } + + fn children_mut<'a>(&'a self) -> Box> + 'a> { + Box::new(std::iter::once(&mut self.input)) + } +} + #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct WindowPartitionTopN { pub func: WindowPartitionTopNFunc, From e23c2d7163e77bd60eca48357f9e5e09e24a269a Mon Sep 17 00:00:00 2001 From: zhang2014 Date: Sat, 28 Jun 2025 13:05:18 +0800 Subject: [PATCH 2/8] refactor(query): use trait to refactor physical plan --- src/query/sql/src/executor/physical_plan.rs | 655 +----------------- .../physical_add_stream_column.rs | 23 +- .../physical_aggregate_expand.rs | 30 +- .../physical_aggregate_final.rs | 21 +- .../physical_aggregate_partial.rs | 26 +- .../physical_plans/physical_async_func.rs | 36 +- .../physical_plans/physical_broadcast.rs | 36 +- .../physical_plans/physical_cache_scan.rs | 13 +- .../physical_column_mutation.rs | 30 +- .../physical_plans/physical_commit_sink.rs | 27 +- .../physical_plans/physical_compact_source.rs | 13 +- .../physical_constant_table_scan.rs | 15 +- .../physical_copy_into_location.rs | 30 +- .../physical_copy_into_table.rs | 39 +- .../physical_distributed_insert_select.rs | 30 +- .../physical_plans/physical_eval_scalar.rs | 29 +- .../physical_plans/physical_exchange.rs | 31 +- .../physical_plans/physical_exchange_sink.rs | 33 +- .../physical_exchange_source.rs | 18 +- .../physical_expression_scan.rs | 23 +- .../physical_plans/physical_filter.rs | 15 +- .../physical_plans/physical_hash_join.rs | 12 +- .../executor/physical_plans/physical_limit.rs | 8 +- .../physical_multi_table_insert.rs | 187 ++++- .../physical_plans/physical_mutation.rs | 25 +- .../physical_mutation_into_organize.rs | 21 +- .../physical_mutation_into_split.rs | 21 +- .../physical_mutation_manipulate.rs | 21 +- .../physical_mutation_source.rs | 19 +- .../physical_plans/physical_project_set.rs | 21 +- .../physical_plans/physical_r_cte_scan.rs | 13 +- .../physical_plans/physical_range_join.rs | 29 +- .../physical_plans/physical_recluster.rs | 34 +- .../physical_replace_async_source.rs | 10 +- .../physical_replace_deduplicate.rs | 26 +- .../physical_plans/physical_replace_into.rs | 27 +- .../physical_plans/physical_row_fetch.rs | 9 +- .../executor/physical_plans/physical_sort.rs | 44 +- .../physical_plans/physical_table_scan.rs | 31 +- .../executor/physical_plans/physical_udf.rs | 30 +- .../physical_plans/physical_union_all.rs | 21 +- .../physical_plans/physical_window.rs | 16 +- .../physical_window_partition.rs | 12 +- 43 files changed, 804 insertions(+), 1006 deletions(-) diff --git a/src/query/sql/src/executor/physical_plan.rs b/src/query/sql/src/executor/physical_plan.rs index 8f8fd4c30f335..cec50f8837928 100644 --- a/src/query/sql/src/executor/physical_plan.rs +++ b/src/query/sql/src/executor/physical_plan.rs @@ -124,6 +124,9 @@ pub trait IPhysicalPlan: serde::Serialize + serde::Deserialize + Clone + Debug { self.get_meta().name.clone() } + /// Adjust the plan_id of the physical plan. + /// This function will assign a unique plan_id to each physical plan node in a top-down manner. + /// Which means the plan_id of a node is always greater than the plan_id of its parent node. #[recursive::recursive] fn adjust_plan_id(&mut self, next_id: &mut u32) { self.get_meta_mut().plan_id = *next_id; @@ -149,6 +152,11 @@ pub trait IPhysicalPlan: serde::Serialize + serde::Deserialize + Clone + Debug { Box::new(std::iter::empty()) } + /// Used to find data source info in a non-aggregation and single-table query plan. + fn try_find_single_data_source(&self) -> Option<&DataSourcePlan> { + None + } + #[recursive::recursive] fn try_find_mutation_source(&self) -> Option { for child in self.children() { @@ -177,10 +185,6 @@ pub trait IPhysicalPlan: serde::Serialize + serde::Deserialize + Clone + Debug { #[recursive::recursive] fn is_distributed_plan(&self) -> bool { self.children().any(|child| child.is_distributed_plan()) - // || matches!( - // self, - // Self::ExchangeSource(_) | Self::ExchangeSink(_) | Self::Exchange(_) - // ) } #[recursive::recursive] @@ -275,382 +279,6 @@ pub enum PhysicalPlan { } impl PhysicalPlan { - /// Adjust the plan_id of the physical plan. - /// This function will assign a unique plan_id to each physical plan node in a top-down manner. - /// Which means the plan_id of a node is always greater than the plan_id of its parent node. - // #[recursive::recursive] - // pub fn adjust_plan_id(&mut self, next_id: &mut u32) { - // match self { - // PhysicalPlan::AsyncFunction(plan) => { - // plan.plan_id = *next_id; - // *next_id += 1; - // } - // PhysicalPlan::TableScan(plan) => { - // plan.plan_id = *next_id; - // *next_id += 1; - // } - // PhysicalPlan::Filter(plan) => { - // plan.plan_id = *next_id; - // *next_id += 1; - // plan.input.adjust_plan_id(next_id); - // } - // PhysicalPlan::EvalScalar(plan) => { - // plan.plan_id = *next_id; - // *next_id += 1; - // plan.input.adjust_plan_id(next_id); - // } - // PhysicalPlan::ProjectSet(plan) => { - // plan.plan_id = *next_id; - // *next_id += 1; - // plan.input.adjust_plan_id(next_id); - // } - // PhysicalPlan::AggregateExpand(plan) => { - // plan.plan_id = *next_id; - // *next_id += 1; - // plan.input.adjust_plan_id(next_id); - // } - // PhysicalPlan::AggregatePartial(plan) => { - // plan.plan_id = *next_id; - // *next_id += 1; - // plan.input.adjust_plan_id(next_id); - // } - // PhysicalPlan::AggregateFinal(plan) => { - // plan.plan_id = *next_id; - // *next_id += 1; - // plan.input.adjust_plan_id(next_id); - // } - // PhysicalPlan::Window(plan) => { - // plan.plan_id = *next_id; - // *next_id += 1; - // plan.input.adjust_plan_id(next_id); - // } - // PhysicalPlan::WindowPartition(plan) => { - // plan.plan_id = *next_id; - // *next_id += 1; - // plan.input.adjust_plan_id(next_id); - // } - // PhysicalPlan::Sort(plan) => { - // plan.plan_id = *next_id; - // *next_id += 1; - // plan.input.adjust_plan_id(next_id); - // } - // PhysicalPlan::Limit(plan) => { - // plan.plan_id = *next_id; - // *next_id += 1; - // plan.input.adjust_plan_id(next_id); - // } - // PhysicalPlan::RowFetch(plan) => { - // plan.plan_id = *next_id; - // *next_id += 1; - // plan.input.adjust_plan_id(next_id); - // } - // PhysicalPlan::HashJoin(plan) => { - // plan.plan_id = *next_id; - // *next_id += 1; - // plan.probe.adjust_plan_id(next_id); - // plan.build.adjust_plan_id(next_id); - // } - // PhysicalPlan::RangeJoin(plan) => { - // plan.plan_id = *next_id; - // *next_id += 1; - // plan.left.adjust_plan_id(next_id); - // plan.right.adjust_plan_id(next_id); - // } - // PhysicalPlan::Exchange(plan) => { - // plan.plan_id = *next_id; - // *next_id += 1; - // plan.input.adjust_plan_id(next_id); - // } - // PhysicalPlan::UnionAll(plan) => { - // plan.plan_id = *next_id; - // *next_id += 1; - // plan.left.adjust_plan_id(next_id); - // plan.right.adjust_plan_id(next_id); - // } - // PhysicalPlan::RecursiveCteScan(plan) => { - // plan.plan_id = *next_id; - // *next_id += 1; - // } - // PhysicalPlan::ConstantTableScan(plan) => { - // plan.plan_id = *next_id; - // *next_id += 1; - // } - // PhysicalPlan::ExpressionScan(plan) => { - // plan.plan_id = *next_id; - // *next_id += 1; - // } - // PhysicalPlan::CacheScan(plan) => { - // plan.plan_id = *next_id; - // *next_id += 1; - // } - // PhysicalPlan::Udf(plan) => { - // plan.plan_id = *next_id; - // *next_id += 1; - // plan.input.adjust_plan_id(next_id); - // } - // PhysicalPlan::DistributedInsertSelect(plan) => { - // plan.plan_id = *next_id; - // *next_id += 1; - // plan.input.adjust_plan_id(next_id); - // } - // PhysicalPlan::ExchangeSource(plan) => { - // plan.plan_id = *next_id; - // *next_id += 1; - // } - // PhysicalPlan::ExchangeSink(plan) => { - // plan.plan_id = *next_id; - // *next_id += 1; - // } - // PhysicalPlan::CopyIntoTable(plan) => { - // plan.plan_id = *next_id; - // *next_id += 1; - // match &mut plan.source { - // CopyIntoTableSource::Query(input) => input.adjust_plan_id(next_id), - // CopyIntoTableSource::Stage(input) => input.adjust_plan_id(next_id), - // }; - // } - // PhysicalPlan::CopyIntoLocation(plan) => { - // plan.plan_id = *next_id; - // *next_id += 1; - // plan.input.adjust_plan_id(next_id); - // } - // PhysicalPlan::ReplaceInto(plan) => { - // plan.plan_id = *next_id; - // *next_id += 1; - // plan.input.adjust_plan_id(next_id); - // } - // PhysicalPlan::MutationSource(plan) => { - // plan.plan_id = *next_id; - // *next_id += 1; - // } - // PhysicalPlan::ColumnMutation(plan) => { - // plan.plan_id = *next_id; - // *next_id += 1; - // plan.input.adjust_plan_id(next_id); - // } - // PhysicalPlan::Mutation(plan) => { - // plan.plan_id = *next_id; - // *next_id += 1; - // plan.input.adjust_plan_id(next_id); - // } - // PhysicalPlan::MutationSplit(plan) => { - // plan.plan_id = *next_id; - // *next_id += 1; - // plan.input.adjust_plan_id(next_id); - // } - // PhysicalPlan::MutationManipulate(plan) => { - // plan.plan_id = *next_id; - // *next_id += 1; - // plan.input.adjust_plan_id(next_id); - // } - // PhysicalPlan::MutationOrganize(plan) => { - // plan.plan_id = *next_id; - // *next_id += 1; - // plan.input.adjust_plan_id(next_id); - // } - // PhysicalPlan::AddStreamColumn(plan) => { - // plan.plan_id = *next_id; - // *next_id += 1; - // plan.input.adjust_plan_id(next_id); - // } - // PhysicalPlan::CommitSink(plan) => { - // plan.plan_id = *next_id; - // *next_id += 1; - // plan.input.adjust_plan_id(next_id); - // } - // PhysicalPlan::ReplaceAsyncSourcer(plan) => { - // plan.plan_id = *next_id; - // *next_id += 1; - // } - // PhysicalPlan::ReplaceDeduplicate(plan) => { - // plan.plan_id = *next_id; - // *next_id += 1; - // plan.input.adjust_plan_id(next_id); - // } - // PhysicalPlan::CompactSource(plan) => { - // plan.plan_id = *next_id; - // *next_id += 1; - // } - // PhysicalPlan::Recluster(plan) => { - // plan.plan_id = *next_id; - // *next_id += 1; - // } - // PhysicalPlan::HilbertPartition(plan) => { - // plan.plan_id = *next_id; - // *next_id += 1; - // } - // PhysicalPlan::Duplicate(plan) => { - // plan.plan_id = *next_id; - // *next_id += 1; - // plan.input.adjust_plan_id(next_id); - // } - // PhysicalPlan::Shuffle(plan) => { - // plan.plan_id = *next_id; - // *next_id += 1; - // plan.input.adjust_plan_id(next_id); - // } - // PhysicalPlan::ChunkFilter(plan) => { - // plan.plan_id = *next_id; - // *next_id += 1; - // plan.input.adjust_plan_id(next_id); - // } - // PhysicalPlan::ChunkEvalScalar(plan) => { - // plan.plan_id = *next_id; - // *next_id += 1; - // plan.input.adjust_plan_id(next_id); - // } - // PhysicalPlan::ChunkCastSchema(plan) => { - // plan.plan_id = *next_id; - // *next_id += 1; - // plan.input.adjust_plan_id(next_id); - // } - // PhysicalPlan::ChunkFillAndReorder(plan) => { - // plan.plan_id = *next_id; - // *next_id += 1; - // plan.input.adjust_plan_id(next_id); - // } - // PhysicalPlan::ChunkAppendData(plan) => { - // plan.plan_id = *next_id; - // *next_id += 1; - // plan.input.adjust_plan_id(next_id); - // } - // PhysicalPlan::ChunkMerge(plan) => { - // plan.plan_id = *next_id; - // *next_id += 1; - // plan.input.adjust_plan_id(next_id); - // } - // PhysicalPlan::ChunkCommitInsert(plan) => { - // plan.plan_id = *next_id; - // *next_id += 1; - // plan.input.adjust_plan_id(next_id); - // } - // PhysicalPlan::BroadcastSource(plan) => { - // plan.plan_id = *next_id; - // *next_id += 1; - // } - // PhysicalPlan::BroadcastSink(plan) => { - // plan.plan_id = *next_id; - // *next_id += 1; - // plan.input.adjust_plan_id(next_id); - // } - // } - // } - - /// Get the id of the plan node - // pub fn get_id(&self) -> u32 { - // match self { - // PhysicalPlan::AsyncFunction(v) => v.plan_id, - // PhysicalPlan::TableScan(v) => v.plan_id, - // PhysicalPlan::Filter(v) => v.plan_id, - // PhysicalPlan::EvalScalar(v) => v.plan_id, - // PhysicalPlan::ProjectSet(v) => v.plan_id, - // PhysicalPlan::AggregateExpand(v) => v.plan_id, - // PhysicalPlan::AggregatePartial(v) => v.plan_id, - // PhysicalPlan::AggregateFinal(v) => v.plan_id, - // PhysicalPlan::Window(v) => v.plan_id, - // PhysicalPlan::WindowPartition(v) => v.plan_id, - // PhysicalPlan::Sort(v) => v.plan_id, - // PhysicalPlan::Limit(v) => v.plan_id, - // PhysicalPlan::RowFetch(v) => v.plan_id, - // PhysicalPlan::HashJoin(v) => v.plan_id, - // PhysicalPlan::RangeJoin(v) => v.plan_id, - // PhysicalPlan::Exchange(v) => v.plan_id, - // PhysicalPlan::UnionAll(v) => v.plan_id, - // PhysicalPlan::DistributedInsertSelect(v) => v.plan_id, - // PhysicalPlan::ExchangeSource(v) => v.plan_id, - // PhysicalPlan::ExchangeSink(v) => v.plan_id, - // PhysicalPlan::ConstantTableScan(v) => v.plan_id, - // PhysicalPlan::ExpressionScan(v) => v.plan_id, - // PhysicalPlan::CacheScan(v) => v.plan_id, - // PhysicalPlan::Udf(v) => v.plan_id, - // PhysicalPlan::MutationSource(v) => v.plan_id, - // PhysicalPlan::ColumnMutation(v) => v.plan_id, - // PhysicalPlan::Mutation(v) => v.plan_id, - // PhysicalPlan::MutationSplit(v) => v.plan_id, - // PhysicalPlan::MutationManipulate(v) => v.plan_id, - // PhysicalPlan::MutationOrganize(v) => v.plan_id, - // PhysicalPlan::AddStreamColumn(v) => v.plan_id, - // PhysicalPlan::CommitSink(v) => v.plan_id, - // PhysicalPlan::CopyIntoTable(v) => v.plan_id, - // PhysicalPlan::CopyIntoLocation(v) => v.plan_id, - // PhysicalPlan::ReplaceAsyncSourcer(v) => v.plan_id, - // PhysicalPlan::ReplaceDeduplicate(v) => v.plan_id, - // PhysicalPlan::ReplaceInto(v) => v.plan_id, - // PhysicalPlan::CompactSource(v) => v.plan_id, - // PhysicalPlan::Recluster(v) => v.plan_id, - // PhysicalPlan::HilbertPartition(v) => v.plan_id, - // PhysicalPlan::Duplicate(v) => v.plan_id, - // PhysicalPlan::Shuffle(v) => v.plan_id, - // PhysicalPlan::ChunkFilter(v) => v.plan_id, - // PhysicalPlan::ChunkEvalScalar(v) => v.plan_id, - // PhysicalPlan::ChunkCastSchema(v) => v.plan_id, - // PhysicalPlan::ChunkFillAndReorder(v) => v.plan_id, - // PhysicalPlan::ChunkAppendData(v) => v.plan_id, - // PhysicalPlan::ChunkMerge(v) => v.plan_id, - // PhysicalPlan::ChunkCommitInsert(v) => v.plan_id, - // PhysicalPlan::RecursiveCteScan(v) => v.plan_id, - // PhysicalPlan::BroadcastSource(v) => v.plan_id, - // PhysicalPlan::BroadcastSink(v) => v.plan_id, - // } - // } - - pub fn output_schema(&self) -> Result { - match self { - PhysicalPlan::AsyncFunction(plan) => plan.output_schema(), - PhysicalPlan::TableScan(plan) => plan.output_schema(), - PhysicalPlan::Filter(plan) => plan.output_schema(), - PhysicalPlan::EvalScalar(plan) => plan.output_schema(), - PhysicalPlan::AggregateExpand(plan) => plan.output_schema(), - PhysicalPlan::AggregatePartial(plan) => plan.output_schema(), - PhysicalPlan::AggregateFinal(plan) => plan.output_schema(), - PhysicalPlan::Window(plan) => plan.output_schema(), - PhysicalPlan::WindowPartition(plan) => plan.output_schema(), - PhysicalPlan::Sort(plan) => plan.output_schema(), - PhysicalPlan::Limit(plan) => plan.output_schema(), - PhysicalPlan::RowFetch(plan) => plan.output_schema(), - PhysicalPlan::HashJoin(plan) => plan.output_schema(), - PhysicalPlan::Exchange(plan) => plan.output_schema(), - PhysicalPlan::ExchangeSource(plan) => plan.output_schema(), - PhysicalPlan::ExchangeSink(plan) => plan.output_schema(), - PhysicalPlan::UnionAll(plan) => plan.output_schema(), - PhysicalPlan::ProjectSet(plan) => plan.output_schema(), - PhysicalPlan::RangeJoin(plan) => plan.output_schema(), - PhysicalPlan::CopyIntoTable(plan) => plan.output_schema(), - PhysicalPlan::CopyIntoLocation(plan) => plan.output_schema(), - PhysicalPlan::ConstantTableScan(plan) => plan.output_schema(), - PhysicalPlan::ExpressionScan(plan) => plan.output_schema(), - PhysicalPlan::CacheScan(plan) => plan.output_schema(), - PhysicalPlan::RecursiveCteScan(plan) => plan.output_schema(), - PhysicalPlan::Udf(plan) => plan.output_schema(), - PhysicalPlan::MutationSource(plan) => plan.output_schema(), - PhysicalPlan::MutationSplit(plan) => plan.output_schema(), - PhysicalPlan::MutationManipulate(plan) => plan.output_schema(), - PhysicalPlan::MutationOrganize(plan) => plan.output_schema(), - PhysicalPlan::AddStreamColumn(plan) => plan.output_schema(), - PhysicalPlan::Mutation(_) - | PhysicalPlan::ColumnMutation(_) - | PhysicalPlan::ReplaceAsyncSourcer(_) - | PhysicalPlan::ReplaceDeduplicate(_) - | PhysicalPlan::ReplaceInto(_) - | PhysicalPlan::CompactSource(_) - | PhysicalPlan::CommitSink(_) - | PhysicalPlan::DistributedInsertSelect(_) - | PhysicalPlan::Recluster(_) - | PhysicalPlan::BroadcastSource(_) - | PhysicalPlan::BroadcastSink(_) - | PhysicalPlan::HilbertPartition(_) => Ok(DataSchemaRef::default()), - PhysicalPlan::Duplicate(plan) => plan.input.output_schema(), - PhysicalPlan::Shuffle(plan) => plan.input.output_schema(), - PhysicalPlan::ChunkFilter(plan) => plan.input.output_schema(), - PhysicalPlan::ChunkEvalScalar(_) => todo!(), - PhysicalPlan::ChunkCastSchema(_) => todo!(), - PhysicalPlan::ChunkFillAndReorder(_) => todo!(), - PhysicalPlan::ChunkAppendData(_) => todo!(), - PhysicalPlan::ChunkMerge(_) => todo!(), - PhysicalPlan::ChunkCommitInsert(_) => todo!(), - } - } - pub fn name(&self) -> String { match self { PhysicalPlan::TableScan(v) => match &v.source.source_info { @@ -714,229 +342,6 @@ impl PhysicalPlan { } } - pub fn children<'a>(&'a self) -> Box + 'a> { - match self { - PhysicalPlan::TableScan(_) - | PhysicalPlan::ConstantTableScan(_) - | PhysicalPlan::CacheScan(_) - | PhysicalPlan::ExchangeSource(_) - | PhysicalPlan::CompactSource(_) - | PhysicalPlan::ReplaceAsyncSourcer(_) - | PhysicalPlan::Recluster(_) - | PhysicalPlan::RecursiveCteScan(_) - | PhysicalPlan::BroadcastSource(_) => Box::new(std::iter::empty()), - PhysicalPlan::HilbertPartition(plan) => Box::new(std::iter::once(plan.input.as_ref())), - PhysicalPlan::Filter(plan) => Box::new(std::iter::once(plan.input.as_ref())), - PhysicalPlan::EvalScalar(plan) => Box::new(std::iter::once(plan.input.as_ref())), - PhysicalPlan::AggregateExpand(plan) => Box::new(std::iter::once(plan.input.as_ref())), - PhysicalPlan::AggregatePartial(plan) => Box::new(std::iter::once(plan.input.as_ref())), - PhysicalPlan::AggregateFinal(plan) => Box::new(std::iter::once(plan.input.as_ref())), - PhysicalPlan::Window(plan) => Box::new(std::iter::once(plan.input.as_ref())), - PhysicalPlan::WindowPartition(plan) => Box::new(std::iter::once(plan.input.as_ref())), - PhysicalPlan::Sort(plan) => Box::new(std::iter::once(plan.input.as_ref())), - PhysicalPlan::Limit(plan) => Box::new(std::iter::once(plan.input.as_ref())), - PhysicalPlan::RowFetch(plan) => Box::new(std::iter::once(plan.input.as_ref())), - PhysicalPlan::HashJoin(plan) => Box::new( - std::iter::once(plan.probe.as_ref()).chain(std::iter::once(plan.build.as_ref())), - ), - PhysicalPlan::ExpressionScan(plan) => Box::new(std::iter::once(plan.input.as_ref())), - PhysicalPlan::Exchange(plan) => Box::new(std::iter::once(plan.input.as_ref())), - PhysicalPlan::ExchangeSink(plan) => Box::new(std::iter::once(plan.input.as_ref())), - PhysicalPlan::BroadcastSink(plan) => Box::new(std::iter::once(plan.input.as_ref())), - PhysicalPlan::UnionAll(plan) => Box::new( - std::iter::once(plan.left.as_ref()).chain(std::iter::once(plan.right.as_ref())), - ), - PhysicalPlan::DistributedInsertSelect(plan) => { - Box::new(std::iter::once(plan.input.as_ref())) - } - PhysicalPlan::CommitSink(plan) => Box::new(std::iter::once(plan.input.as_ref())), - PhysicalPlan::ProjectSet(plan) => Box::new(std::iter::once(plan.input.as_ref())), - PhysicalPlan::RangeJoin(plan) => Box::new( - std::iter::once(plan.left.as_ref()).chain(std::iter::once(plan.right.as_ref())), - ), - PhysicalPlan::ReplaceDeduplicate(plan) => { - Box::new(std::iter::once(plan.input.as_ref())) - } - PhysicalPlan::ReplaceInto(plan) => Box::new(std::iter::once(plan.input.as_ref())), - PhysicalPlan::MutationSource(_) => Box::new(std::iter::empty()), - PhysicalPlan::ColumnMutation(plan) => Box::new(std::iter::once(plan.input.as_ref())), - PhysicalPlan::Mutation(plan) => Box::new(std::iter::once(plan.input.as_ref())), - PhysicalPlan::MutationSplit(plan) => Box::new(std::iter::once(plan.input.as_ref())), - PhysicalPlan::MutationManipulate(plan) => { - Box::new(std::iter::once(plan.input.as_ref())) - } - PhysicalPlan::MutationOrganize(plan) => Box::new(std::iter::once(plan.input.as_ref())), - PhysicalPlan::AddStreamColumn(plan) => Box::new(std::iter::once(plan.input.as_ref())), - PhysicalPlan::Udf(plan) => Box::new(std::iter::once(plan.input.as_ref())), - PhysicalPlan::AsyncFunction(plan) => Box::new(std::iter::once(plan.input.as_ref())), - PhysicalPlan::CopyIntoLocation(plan) => Box::new(std::iter::once(plan.input.as_ref())), - PhysicalPlan::Duplicate(plan) => Box::new(std::iter::once(plan.input.as_ref())), - PhysicalPlan::Shuffle(plan) => Box::new(std::iter::once(plan.input.as_ref())), - PhysicalPlan::ChunkFilter(plan) => Box::new(std::iter::once(plan.input.as_ref())), - PhysicalPlan::ChunkEvalScalar(plan) => Box::new(std::iter::once(plan.input.as_ref())), - PhysicalPlan::ChunkCastSchema(plan) => Box::new(std::iter::once(plan.input.as_ref())), - PhysicalPlan::ChunkFillAndReorder(plan) => { - Box::new(std::iter::once(plan.input.as_ref())) - } - PhysicalPlan::ChunkAppendData(plan) => Box::new(std::iter::once(plan.input.as_ref())), - PhysicalPlan::ChunkMerge(plan) => Box::new(std::iter::once(plan.input.as_ref())), - PhysicalPlan::ChunkCommitInsert(plan) => Box::new(std::iter::once(plan.input.as_ref())), - PhysicalPlan::CopyIntoTable(v) => match &v.source { - CopyIntoTableSource::Query(v) => Box::new(std::iter::once(v.as_ref())), - CopyIntoTableSource::Stage(v) => Box::new(std::iter::once(v.as_ref())), - }, - } - } - - pub fn children_mut<'a>(&'a mut self) -> Box + 'a> { - match self { - PhysicalPlan::TableScan(_) - | PhysicalPlan::ConstantTableScan(_) - | PhysicalPlan::CacheScan(_) - | PhysicalPlan::ExchangeSource(_) - | PhysicalPlan::CompactSource(_) - | PhysicalPlan::ReplaceAsyncSourcer(_) - | PhysicalPlan::Recluster(_) - | PhysicalPlan::BroadcastSource(_) - | PhysicalPlan::RecursiveCteScan(_) => Box::new(std::iter::empty()), - PhysicalPlan::HilbertPartition(plan) => Box::new(std::iter::once(plan.input.as_mut())), - PhysicalPlan::Filter(plan) => Box::new(std::iter::once(plan.input.as_mut())), - PhysicalPlan::EvalScalar(plan) => Box::new(std::iter::once(plan.input.as_mut())), - PhysicalPlan::AggregateExpand(plan) => Box::new(std::iter::once(plan.input.as_mut())), - PhysicalPlan::AggregatePartial(plan) => Box::new(std::iter::once(plan.input.as_mut())), - PhysicalPlan::AggregateFinal(plan) => Box::new(std::iter::once(plan.input.as_mut())), - PhysicalPlan::Window(plan) => Box::new(std::iter::once(plan.input.as_mut())), - PhysicalPlan::WindowPartition(plan) => Box::new(std::iter::once(plan.input.as_mut())), - PhysicalPlan::Sort(plan) => Box::new(std::iter::once(plan.input.as_mut())), - PhysicalPlan::Limit(plan) => Box::new(std::iter::once(plan.input.as_mut())), - PhysicalPlan::RowFetch(plan) => Box::new(std::iter::once(plan.input.as_mut())), - PhysicalPlan::HashJoin(plan) => Box::new( - std::iter::once(plan.probe.as_mut()).chain(std::iter::once(plan.build.as_mut())), - ), - PhysicalPlan::ExpressionScan(plan) => Box::new(std::iter::once(plan.input.as_mut())), - PhysicalPlan::Exchange(plan) => Box::new(std::iter::once(plan.input.as_mut())), - PhysicalPlan::ExchangeSink(plan) => Box::new(std::iter::once(plan.input.as_mut())), - PhysicalPlan::UnionAll(plan) => Box::new( - std::iter::once(plan.left.as_mut()).chain(std::iter::once(plan.right.as_mut())), - ), - PhysicalPlan::DistributedInsertSelect(plan) => { - Box::new(std::iter::once(plan.input.as_mut())) - } - PhysicalPlan::CommitSink(plan) => Box::new(std::iter::once(plan.input.as_mut())), - PhysicalPlan::ProjectSet(plan) => Box::new(std::iter::once(plan.input.as_mut())), - PhysicalPlan::RangeJoin(plan) => Box::new( - std::iter::once(plan.left.as_mut()).chain(std::iter::once(plan.right.as_mut())), - ), - PhysicalPlan::ReplaceDeduplicate(plan) => { - Box::new(std::iter::once(plan.input.as_mut())) - } - PhysicalPlan::ReplaceInto(plan) => Box::new(std::iter::once(plan.input.as_mut())), - PhysicalPlan::MutationSource(_) => Box::new(std::iter::empty()), - PhysicalPlan::ColumnMutation(plan) => Box::new(std::iter::once(plan.input.as_mut())), - PhysicalPlan::Mutation(plan) => Box::new(std::iter::once(plan.input.as_mut())), - PhysicalPlan::MutationSplit(plan) => Box::new(std::iter::once(plan.input.as_mut())), - PhysicalPlan::MutationManipulate(plan) => { - Box::new(std::iter::once(plan.input.as_mut())) - } - PhysicalPlan::MutationOrganize(plan) => Box::new(std::iter::once(plan.input.as_mut())), - PhysicalPlan::AddStreamColumn(plan) => Box::new(std::iter::once(plan.input.as_mut())), - PhysicalPlan::Udf(plan) => Box::new(std::iter::once(plan.input.as_mut())), - PhysicalPlan::AsyncFunction(plan) => Box::new(std::iter::once(plan.input.as_mut())), - PhysicalPlan::CopyIntoLocation(plan) => Box::new(std::iter::once(plan.input.as_mut())), - PhysicalPlan::Duplicate(plan) => Box::new(std::iter::once(plan.input.as_mut())), - PhysicalPlan::Shuffle(plan) => Box::new(std::iter::once(plan.input.as_mut())), - PhysicalPlan::ChunkFilter(plan) => Box::new(std::iter::once(plan.input.as_mut())), - PhysicalPlan::ChunkEvalScalar(plan) => Box::new(std::iter::once(plan.input.as_mut())), - PhysicalPlan::ChunkCastSchema(plan) => Box::new(std::iter::once(plan.input.as_mut())), - PhysicalPlan::ChunkFillAndReorder(plan) => { - Box::new(std::iter::once(plan.input.as_mut())) - } - PhysicalPlan::ChunkAppendData(plan) => Box::new(std::iter::once(plan.input.as_mut())), - PhysicalPlan::ChunkMerge(plan) => Box::new(std::iter::once(plan.input.as_mut())), - PhysicalPlan::ChunkCommitInsert(plan) => Box::new(std::iter::once(plan.input.as_mut())), - PhysicalPlan::CopyIntoTable(v) => match &mut v.source { - CopyIntoTableSource::Query(v) => Box::new(std::iter::once(v.as_mut())), - CopyIntoTableSource::Stage(v) => Box::new(std::iter::once(v.as_mut())), - }, - PhysicalPlan::BroadcastSink(plan) => Box::new(std::iter::once(plan.input.as_mut())), - } - } - - /// Used to find data source info in a non-aggregation and single-table query plan. - #[recursive::recursive] - pub fn try_find_single_data_source(&self) -> Option<&DataSourcePlan> { - match self { - PhysicalPlan::TableScan(scan) => Some(&scan.source), - PhysicalPlan::Filter(plan) => plan.input.try_find_single_data_source(), - PhysicalPlan::EvalScalar(plan) => plan.input.try_find_single_data_source(), - PhysicalPlan::Window(plan) => plan.input.try_find_single_data_source(), - PhysicalPlan::WindowPartition(plan) => plan.input.try_find_single_data_source(), - PhysicalPlan::Sort(plan) => plan.input.try_find_single_data_source(), - PhysicalPlan::Limit(plan) => plan.input.try_find_single_data_source(), - PhysicalPlan::Exchange(plan) => plan.input.try_find_single_data_source(), - PhysicalPlan::ExchangeSink(plan) => plan.input.try_find_single_data_source(), - PhysicalPlan::DistributedInsertSelect(plan) => plan.input.try_find_single_data_source(), - PhysicalPlan::ProjectSet(plan) => plan.input.try_find_single_data_source(), - PhysicalPlan::RowFetch(plan) => plan.input.try_find_single_data_source(), - PhysicalPlan::Udf(plan) => plan.input.try_find_single_data_source(), - PhysicalPlan::AsyncFunction(plan) => plan.input.try_find_single_data_source(), - PhysicalPlan::CopyIntoLocation(plan) => plan.input.try_find_single_data_source(), - PhysicalPlan::UnionAll(_) - | PhysicalPlan::ExchangeSource(_) - | PhysicalPlan::HashJoin(_) - | PhysicalPlan::RangeJoin(_) - | PhysicalPlan::AggregateExpand(_) - | PhysicalPlan::AggregateFinal(_) - | PhysicalPlan::AggregatePartial(_) - | PhysicalPlan::CompactSource(_) - | PhysicalPlan::CommitSink(_) - | PhysicalPlan::CopyIntoTable(_) - | PhysicalPlan::ReplaceAsyncSourcer(_) - | PhysicalPlan::ReplaceDeduplicate(_) - | PhysicalPlan::ReplaceInto(_) - | PhysicalPlan::MutationSource(_) - | PhysicalPlan::ColumnMutation(_) - | PhysicalPlan::Mutation(_) - | PhysicalPlan::MutationSplit(_) - | PhysicalPlan::MutationManipulate(_) - | PhysicalPlan::MutationOrganize(_) - | PhysicalPlan::AddStreamColumn(_) - | PhysicalPlan::ConstantTableScan(_) - | PhysicalPlan::ExpressionScan(_) - | PhysicalPlan::CacheScan(_) - | PhysicalPlan::RecursiveCteScan(_) - | PhysicalPlan::Recluster(_) - | PhysicalPlan::HilbertPartition(_) - | PhysicalPlan::Duplicate(_) - | PhysicalPlan::Shuffle(_) - | PhysicalPlan::ChunkFilter(_) - | PhysicalPlan::ChunkEvalScalar(_) - | PhysicalPlan::ChunkCastSchema(_) - | PhysicalPlan::ChunkFillAndReorder(_) - | PhysicalPlan::ChunkAppendData(_) - | PhysicalPlan::ChunkMerge(_) - | PhysicalPlan::ChunkCommitInsert(_) - | PhysicalPlan::BroadcastSource(_) - | PhysicalPlan::BroadcastSink(_) => None, - } - } - - #[recursive::recursive] - pub fn is_distributed_plan(&self) -> bool { - self.children().any(|child| child.is_distributed_plan()) - || matches!( - self, - Self::ExchangeSource(_) | Self::ExchangeSink(_) | Self::Exchange(_) - ) - } - - // #[recursive::recursive] - // pub fn is_warehouse_distributed_plan(&self) -> bool { - // self.children() - // .any(|child| child.is_warehouse_distributed_plan()) - // || matches!(self, Self::TableScan(v) if v.source.parts.kind == PartitionsShuffleKind::BroadcastWarehouse) - // } - pub fn get_desc(&self) -> Result { Ok(match self { PhysicalPlan::TableScan(v) => format!( @@ -1204,48 +609,4 @@ impl PhysicalPlan { Ok(labels) } - - #[recursive::recursive] - pub fn try_find_mutation_source(&self) -> Option { - match self { - PhysicalPlan::MutationSource(mutation_source) => Some(mutation_source.clone()), - _ => { - for child in self.children() { - if let Some(plan) = child.try_find_mutation_source() { - return Some(plan); - } - } - None - } - } - } - - // #[recursive::recursive] - // pub fn get_all_data_source(&self, sources: &mut Vec<(u32, Box)>) { - // match self { - // PhysicalPlan::TableScan(table_scan) => { - // sources.push((table_scan.plan_id, table_scan.source.clone())); - // } - // _ => { - // for child in self.children() { - // child.get_all_data_source(sources); - // } - // } - // } - // } - // - // pub fn set_pruning_stats(&mut self, stats: &mut HashMap) { - // match self { - // PhysicalPlan::TableScan(table_scan) => { - // if let Some(stat) = stats.remove(&table_scan.plan_id) { - // table_scan.source.statistics = stat; - // } - // } - // _ => { - // for child in self.children_mut() { - // child.set_pruning_stats(stats) - // } - // } - // } - // } } diff --git a/src/query/sql/src/executor/physical_plans/physical_add_stream_column.rs b/src/query/sql/src/executor/physical_plans/physical_add_stream_column.rs index 3e140a596253c..9c774af387537 100644 --- a/src/query/sql/src/executor/physical_plans/physical_add_stream_column.rs +++ b/src/query/sql/src/executor/physical_plans/physical_add_stream_column.rs @@ -23,7 +23,7 @@ use databend_common_expression::ORIGIN_BLOCK_ID_COL_NAME; use databend_common_expression::ORIGIN_BLOCK_ROW_NUM_COL_NAME; use databend_common_expression::ORIGIN_VERSION_COL_NAME; -use crate::executor::{IPhysicalPlan, PhysicalPlan}; +use crate::executor::{IPhysicalPlan, PhysicalPlan, PhysicalPlanMeta}; use crate::planner::CURRENT_BLOCK_ID_COL_NAME; use crate::planner::CURRENT_BLOCK_ROW_NUM_COL_NAME; use crate::plans::BoundColumnRef; @@ -38,14 +38,29 @@ use crate::Visibility; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct AddStreamColumn { pub plan_id: u32, - pub input: Box, + meta: PhysicalPlanMeta, + pub input: Box, pub exprs: Vec, pub projections: Vec, pub stream_columns: Vec, } impl IPhysicalPlan for AddStreamColumn { + fn get_meta(&self) -> &PhysicalPlanMeta { + &self.meta + } + + fn get_meta_mut(&mut self) -> &mut PhysicalPlanMeta { + &mut self.meta + } + + fn children<'a>(&'a self) -> Box> + 'a> { + Box::new(std::iter::once(&self.input)) + } + fn children_mut<'a>(&'a self) -> Box> + 'a> { + Box::new(std::iter::once(&mut self.input)) + } } impl AddStreamColumn { @@ -168,8 +183,4 @@ impl AddStreamColumn { stream_columns, }) } - - pub fn output_schema(&self) -> Result { - self.input.output_schema() - } } diff --git a/src/query/sql/src/executor/physical_plans/physical_aggregate_expand.rs b/src/query/sql/src/executor/physical_plans/physical_aggregate_expand.rs index 69db39f6daf0a..5e2693f75c07a 100644 --- a/src/query/sql/src/executor/physical_plans/physical_aggregate_expand.rs +++ b/src/query/sql/src/executor/physical_plans/physical_aggregate_expand.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use databend_common_catalog::plan::DataSourcePlan; use databend_common_exception::Result; use databend_common_expression::types::DataType; use databend_common_expression::types::NumberDataType; @@ -82,32 +83,3 @@ impl IPhysicalPlan for AggregateExpand { Box::new(std::iter::once(&mut self.input)) } } - -impl AggregateExpand { - pub fn output_schema(&self) -> Result { - let input_schema = self.input.output_schema()?; - let mut output_fields = input_schema.fields().clone(); - // Add virtual columns to group by. - output_fields.reserve(self.group_bys.len() + 1); - - for (group_by, (actual, ty)) in self - .group_bys - .iter() - .zip(self.grouping_sets.dup_group_items.iter()) - { - // All group by columns will wrap nullable. - let i = input_schema.index_of(&group_by.to_string())?; - let f = &mut output_fields[i]; - debug_assert!(f.data_type() == ty || f.data_type().wrap_nullable() == *ty); - *f = DataField::new(f.name(), f.data_type().wrap_nullable()); - let new_field = DataField::new(&actual.to_string(), ty.clone()); - output_fields.push(new_field); - } - - output_fields.push(DataField::new( - &self.grouping_sets.grouping_id_index.to_string(), - DataType::Number(NumberDataType::UInt32), - )); - Ok(DataSchemaRefExt::create(output_fields)) - } -} diff --git a/src/query/sql/src/executor/physical_plans/physical_aggregate_final.rs b/src/query/sql/src/executor/physical_plans/physical_aggregate_final.rs index af38893493552..54bf09b8afe90 100644 --- a/src/query/sql/src/executor/physical_plans/physical_aggregate_final.rs +++ b/src/query/sql/src/executor/physical_plans/physical_aggregate_final.rs @@ -13,7 +13,7 @@ // limitations under the License. use std::sync::Arc; - +use databend_common_catalog::plan::DataSourcePlan; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_expression::DataField; @@ -87,25 +87,6 @@ impl IPhysicalPlan for AggregateFinal { } } -impl AggregateFinal { - pub fn output_schema(&self) -> Result { - let mut fields = Vec::with_capacity(self.agg_funcs.len() + self.group_by.len()); - for agg in self.agg_funcs.iter() { - let data_type = agg.sig.return_type.clone(); - fields.push(DataField::new(&agg.output_column.to_string(), data_type)); - } - for id in self.group_by.iter() { - let data_type = self - .before_group_by_schema - .field_with_name(&id.to_string())? - .data_type() - .clone(); - fields.push(DataField::new(&id.to_string(), data_type)); - } - Ok(DataSchemaRefExt::create(fields)) - } -} - impl PhysicalPlanBuilder { pub(crate) async fn build_aggregate( &mut self, diff --git a/src/query/sql/src/executor/physical_plans/physical_aggregate_partial.rs b/src/query/sql/src/executor/physical_plans/physical_aggregate_partial.rs index 7727a5889f18f..50156679a8acc 100644 --- a/src/query/sql/src/executor/physical_plans/physical_aggregate_partial.rs +++ b/src/query/sql/src/executor/physical_plans/physical_aggregate_partial.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use databend_common_catalog::plan::DataSourcePlan; use databend_common_exception::Result; use databend_common_expression::types::DataType; #[allow(unused_imports)] @@ -31,7 +32,7 @@ pub struct AggregatePartial { meta: PhysicalPlanMeta, // A unique id of operator in a `PhysicalPlan` tree, only used for display. pub plan_id: u32, - pub input: Box, + pub input: Box, pub group_by: Vec, pub agg_funcs: Vec, pub enable_experimental_aggregate_hashtable: bool, @@ -81,26 +82,3 @@ impl IPhysicalPlan for AggregatePartial { Box::new(std::iter::once(&mut self.input)) } } - -impl AggregatePartial { - pub fn output_schema(&self) -> Result { - let input_schema = self.input.output_schema()?; - - let mut fields = Vec::with_capacity(self.agg_funcs.len() + self.group_by.len()); - - fields.extend(self.agg_funcs.iter().map(|func| { - let name = func.output_column.to_string(); - DataField::new(&name, DataType::Binary) - })); - - for (idx, field) in self.group_by.iter().zip( - self.group_by - .iter() - .map(|index| input_schema.field_with_name(&index.to_string())), - ) { - fields.push(DataField::new(&idx.to_string(), field?.data_type().clone())); - } - - Ok(DataSchemaRefExt::create(fields)) - } -} diff --git a/src/query/sql/src/executor/physical_plans/physical_async_func.rs b/src/query/sql/src/executor/physical_plans/physical_async_func.rs index 05c43a209ddf7..5309bac843d58 100644 --- a/src/query/sql/src/executor/physical_plans/physical_async_func.rs +++ b/src/query/sql/src/executor/physical_plans/physical_async_func.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use databend_common_catalog::plan::DataSourcePlan; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_expression::types::DataType; @@ -20,7 +21,7 @@ use databend_common_expression::DataSchemaRef; use databend_common_expression::DataSchemaRefExt; use crate::executor::explain::PlanStatsInfo; -use crate::executor::{IPhysicalPlan, PhysicalPlan}; +use crate::executor::{IPhysicalPlan, PhysicalPlan, PhysicalPlanMeta}; use crate::executor::PhysicalPlanBuilder; use crate::optimizer::ir::SExpr; use crate::plans::AsyncFunctionArgument; @@ -32,13 +33,22 @@ use crate::ScalarExpr; pub struct AsyncFunction { // A unique id of operator in a `PhysicalPlan` tree, only used for display. pub plan_id: u32, - pub input: Box, + meta: PhysicalPlanMeta, + pub input: Box, pub async_func_descs: Vec, // Only used for explain pub stat_info: Option, } impl IPhysicalPlan for AsyncFunction { + fn get_meta(&self) -> &PhysicalPlanMeta { + &self.meta + } + + fn get_meta_mut(&mut self) -> &mut PhysicalPlanMeta { + &mut self.meta + } + fn output_schema(&self) -> Result { let input_schema = self.input.output_schema()?; let mut fields = input_schema.fields().clone(); @@ -49,18 +59,18 @@ impl IPhysicalPlan for AsyncFunction { } Ok(DataSchemaRefExt::create(fields)) } -} -impl AsyncFunction { - pub fn output_schema(&self) -> Result { - let input_schema = self.input.output_schema()?; - let mut fields = input_schema.fields().clone(); - for async_func_desc in self.async_func_descs.iter() { - let name = async_func_desc.output_column.to_string(); - let data_type = async_func_desc.data_type.clone(); - fields.push(DataField::new(&name, *data_type)); - } - Ok(DataSchemaRefExt::create(fields)) + fn children<'a>(&'a self) -> Box> + 'a> { + Box::new(std::iter::once(&self.input)) + } + + fn children_mut<'a>(&'a self) -> Box> + 'a> { + Box::new(std::iter::once(&mut self.input)) + } + + #[recursive::recursive] + fn try_find_single_data_source(&self) -> Option<&DataSourcePlan> { + self.input.try_find_single_data_source() } } diff --git a/src/query/sql/src/executor/physical_plans/physical_broadcast.rs b/src/query/sql/src/executor/physical_plans/physical_broadcast.rs index 80804392c0753..45a17a3f95a79 100644 --- a/src/query/sql/src/executor/physical_plans/physical_broadcast.rs +++ b/src/query/sql/src/executor/physical_plans/physical_broadcast.rs @@ -14,28 +14,56 @@ use databend_common_catalog::table_context::TableContext; use databend_common_exception::Result; - +use databend_common_expression::DataSchemaRef; use super::Exchange; use super::FragmentKind; -use crate::executor::{IPhysicalPlan, PhysicalPlan}; +use crate::executor::{IPhysicalPlan, PhysicalPlan, PhysicalPlanMeta}; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct BroadcastSource { pub plan_id: u32, + meta: PhysicalPlanMeta, pub broadcast_id: u32, } -impl IPhysicalPlan for BroadcastSource {} +impl IPhysicalPlan for BroadcastSource { + fn get_meta(&self) -> &PhysicalPlanMeta { + &self.meta + } + + fn get_meta_mut(&mut self) -> &mut PhysicalPlanMeta { + &mut self.meta + } +} #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct BroadcastSink { pub plan_id: u32, + meta: PhysicalPlanMeta, pub broadcast_id: u32, - pub input: Box, + pub input: Box, } impl IPhysicalPlan for BroadcastSink { + fn get_meta(&self) -> &PhysicalPlanMeta { + &self.meta + } + fn get_meta_mut(&mut self) -> &mut PhysicalPlanMeta { + &mut self.meta + } + + fn output_schema(&self) -> Result { + Ok(DataSchemaRef::default()) + } + + fn children<'a>(&'a self) -> Box> + 'a> { + Box::new(std::iter::once(&self.input)) + } + + fn children_mut<'a>(&'a self) -> Box> + 'a> { + Box::new(std::iter::once(&mut self.input)) + } } pub fn build_broadcast_plan(broadcast_id: u32) -> Result { diff --git a/src/query/sql/src/executor/physical_plans/physical_cache_scan.rs b/src/query/sql/src/executor/physical_plans/physical_cache_scan.rs index 66e28c83f6082..9bfa453473dc4 100644 --- a/src/query/sql/src/executor/physical_plans/physical_cache_scan.rs +++ b/src/query/sql/src/executor/physical_plans/physical_cache_scan.rs @@ -16,7 +16,7 @@ use databend_common_exception::Result; use databend_common_expression::DataSchemaRef; use databend_common_expression::DataSchemaRefExt; -use crate::executor::{IPhysicalPlan, PhysicalPlan}; +use crate::executor::{IPhysicalPlan, PhysicalPlan, PhysicalPlanMeta}; use crate::executor::PhysicalPlanBuilder; use crate::plans::CacheSource; use crate::ColumnSet; @@ -25,16 +25,21 @@ use crate::ColumnSet; pub struct CacheScan { // A unique id of operator in a `PhysicalPlan` tree, only used for display. pub plan_id: u32, + meta: PhysicalPlanMeta, pub cache_source: CacheSource, pub output_schema: DataSchemaRef, } impl IPhysicalPlan for CacheScan { + fn get_meta(&self) -> &PhysicalPlanMeta { + &self.meta + } -} + fn get_meta_mut(&mut self) -> &mut PhysicalPlanMeta { + &mut self.meta + } -impl CacheScan { - pub fn output_schema(&self) -> Result { + fn output_schema(&self) -> Result { Ok(self.output_schema.clone()) } } diff --git a/src/query/sql/src/executor/physical_plans/physical_column_mutation.rs b/src/query/sql/src/executor/physical_plans/physical_column_mutation.rs index 499a54e8331e3..c413055881e2f 100644 --- a/src/query/sql/src/executor/physical_plans/physical_column_mutation.rs +++ b/src/query/sql/src/executor/physical_plans/physical_column_mutation.rs @@ -13,18 +13,20 @@ // limitations under the License. use std::collections::HashMap; - -use databend_common_expression::RemoteExpr; +use databend_common_catalog::plan::DataSourcePlan; +use databend_common_expression::{DataSchemaRef, RemoteExpr}; use databend_common_meta_app::schema::TableInfo; use databend_storages_common_table_meta::meta::TableMetaTimestamps; -use crate::executor::IPhysicalPlan; +use crate::executor::{IPhysicalPlan, PhysicalPlanMeta}; use crate::executor::physical_plan::PhysicalPlan; use crate::executor::physical_plans::MutationKind; +use databend_common_exception::Result; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct ColumnMutation { pub plan_id: u32, - pub input: Box, + meta: PhysicalPlanMeta, + pub input: Box, pub table_info: TableInfo, pub mutation_expr: Option>, pub computed_expr: Option>, @@ -36,5 +38,23 @@ pub struct ColumnMutation { } impl IPhysicalPlan for ColumnMutation { - + fn get_meta(&self) -> &PhysicalPlanMeta { + &self.meta + } + + fn get_meta_mut(&mut self) -> &mut PhysicalPlanMeta { + &mut self.meta + } + + fn output_schema(&self) -> Result { + Ok(DataSchemaRef::default()) + } + + fn children<'a>(&'a self) -> Box> + 'a> { + Box::new(std::iter::once(&self.input)) + } + + fn children_mut<'a>(&'a self) -> Box> + 'a> { + Box::new(std::iter::once(&mut self.input)) + } } diff --git a/src/query/sql/src/executor/physical_plans/physical_commit_sink.rs b/src/query/sql/src/executor/physical_plans/physical_commit_sink.rs index 88389186eff83..7d8b36e8ec628 100644 --- a/src/query/sql/src/executor/physical_plans/physical_commit_sink.rs +++ b/src/query/sql/src/executor/physical_plans/physical_commit_sink.rs @@ -14,22 +14,25 @@ use std::sync::Arc; -use databend_common_catalog::plan::ReclusterInfoSideCar; +use databend_common_catalog::plan::{DataSourcePlan, ReclusterInfoSideCar}; +use databend_common_expression::DataSchemaRef; use databend_common_meta_app::schema::TableInfo; use databend_common_meta_app::schema::UpdateStreamMetaReq; use databend_storages_common_table_meta::meta::TableMetaTimestamps; use databend_storages_common_table_meta::meta::TableSnapshot; use crate::executor::physical_plans::common::MutationKind; -use crate::executor::{IPhysicalPlan, PhysicalPlan}; +use crate::executor::{IPhysicalPlan, PhysicalPlan, PhysicalPlanMeta}; use crate::plans::TruncateMode; +use databend_common_exception::Result; // serde is required by `PhysicalPlan` /// The commit sink is used to commit the data to the table. #[derive(serde::Serialize, serde::Deserialize, Clone, Debug)] pub struct CommitSink { pub plan_id: u32, - pub input: Box, + meta: PhysicalPlanMeta, + pub input: Box, pub snapshot: Option>, pub table_info: TableInfo, pub commit_type: CommitType, @@ -42,7 +45,25 @@ pub struct CommitSink { } impl IPhysicalPlan for CommitSink { + fn get_meta(&self) -> &PhysicalPlanMeta { + &self.meta + } + fn get_meta_mut(&mut self) -> &mut PhysicalPlanMeta { + &mut self.meta + } + + fn output_schema(&self) -> Result { + Ok(DataSchemaRef::default()) + } + + fn children<'a>(&'a self) -> Box> + 'a> { + Box::new(std::iter::once(&self.input)) + } + + fn children_mut<'a>(&'a self) -> Box> + 'a> { + Box::new(std::iter::once(&mut self.input)) + } } #[derive(serde::Serialize, serde::Deserialize, Clone, Debug)] diff --git a/src/query/sql/src/executor/physical_plans/physical_compact_source.rs b/src/query/sql/src/executor/physical_plans/physical_compact_source.rs index b15e24023385e..84594fbf70d03 100644 --- a/src/query/sql/src/executor/physical_plans/physical_compact_source.rs +++ b/src/query/sql/src/executor/physical_plans/physical_compact_source.rs @@ -14,12 +14,12 @@ use std::collections::HashSet; -use databend_common_catalog::plan::PartInfoType; +use databend_common_catalog::plan::{DataSourcePlan, PartInfoType}; use databend_common_catalog::plan::Partitions; use databend_common_catalog::table::TableExt; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::ColumnId; +use databend_common_expression::{ColumnId, DataSchemaRef}; use databend_common_meta_app::schema::TableInfo; use databend_storages_common_table_meta::meta::TableMetaTimestamps; @@ -28,12 +28,13 @@ use crate::executor::physical_plans::CommitType; use crate::executor::physical_plans::Exchange; use crate::executor::physical_plans::FragmentKind; use crate::executor::physical_plans::MutationKind; -use crate::executor::{IPhysicalPlan, PhysicalPlan}; +use crate::executor::{IPhysicalPlan, PhysicalPlan, PhysicalPlanMeta}; use crate::executor::PhysicalPlanBuilder; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct CompactSource { pub plan_id: u32, + meta: PhysicalPlanMeta, pub parts: Partitions, pub table_info: TableInfo, pub column_ids: HashSet, @@ -41,7 +42,13 @@ pub struct CompactSource { } impl IPhysicalPlan for CompactSource { + fn get_meta(&self) -> &PhysicalPlanMeta { + &self.meta + } + fn get_meta_mut(&mut self) -> &mut PhysicalPlanMeta { + &mut self.meta + } } impl PhysicalPlanBuilder { diff --git a/src/query/sql/src/executor/physical_plans/physical_constant_table_scan.rs b/src/query/sql/src/executor/physical_plans/physical_constant_table_scan.rs index c07b15ae08919..cd1e88aadd9e9 100644 --- a/src/query/sql/src/executor/physical_plans/physical_constant_table_scan.rs +++ b/src/query/sql/src/executor/physical_plans/physical_constant_table_scan.rs @@ -16,7 +16,7 @@ use databend_common_exception::Result; use databend_common_expression::Column; use databend_common_expression::DataSchemaRef; -use crate::executor::{IPhysicalPlan, PhysicalPlan}; +use crate::executor::{IPhysicalPlan, PhysicalPlan, PhysicalPlanMeta}; use crate::executor::PhysicalPlanBuilder; use crate::ColumnSet; use crate::IndexType; @@ -25,20 +25,27 @@ use crate::IndexType; pub struct ConstantTableScan { // A unique id of operator in a `PhysicalPlan` tree, only used for display. pub plan_id: u32, + meta: PhysicalPlanMeta, pub values: Vec, pub num_rows: usize, pub output_schema: DataSchemaRef, } impl IPhysicalPlan for ConstantTableScan { + fn get_meta(&self) -> &PhysicalPlanMeta { + &self.meta + } -} + fn get_meta_mut(&mut self) -> &mut PhysicalPlanMeta { + &mut self.meta + } -impl ConstantTableScan { - pub fn output_schema(&self) -> Result { + fn output_schema(&self) -> Result { Ok(self.output_schema.clone()) } +} +impl ConstantTableScan { pub fn name(&self) -> &str { if self.num_rows == 0 { "EmptyResultScan" diff --git a/src/query/sql/src/executor/physical_plans/physical_copy_into_location.rs b/src/query/sql/src/executor/physical_plans/physical_copy_into_location.rs index 691ef13dc1dc4..4aabfe4300db2 100644 --- a/src/query/sql/src/executor/physical_plans/physical_copy_into_location.rs +++ b/src/query/sql/src/executor/physical_plans/physical_copy_into_location.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use databend_common_catalog::plan::DataSourcePlan; use databend_common_exception::Result; use databend_common_expression::types::DataType; use databend_common_expression::types::NumberDataType; @@ -21,13 +22,14 @@ use databend_common_expression::DataSchemaRefExt; use databend_common_expression::TableSchemaRef; use databend_storages_common_stage::CopyIntoLocationInfo; -use crate::executor::{IPhysicalPlan, PhysicalPlan}; +use crate::executor::{IPhysicalPlan, PhysicalPlan, PhysicalPlanMeta}; use crate::ColumnBinding; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct CopyIntoLocation { pub plan_id: u32, - pub input: Box, + meta: PhysicalPlanMeta, + pub input: Box, pub project_columns: Vec, pub input_data_schema: DataSchemaRef, pub input_table_schema: TableSchemaRef, @@ -35,15 +37,33 @@ pub struct CopyIntoLocation { } impl IPhysicalPlan for CopyIntoLocation { + fn get_meta(&self) -> &PhysicalPlanMeta { + &self.meta + } -} + fn get_meta_mut(&mut self) -> &mut PhysicalPlanMeta { + &mut self.meta + } -impl CopyIntoLocation { - pub fn output_schema(&self) -> Result { + fn output_schema(&self) -> Result { Ok(DataSchemaRefExt::create(vec![ DataField::new("rows_unloaded", DataType::Number(NumberDataType::UInt64)), DataField::new("input_bytes", DataType::Number(NumberDataType::UInt64)), DataField::new("output_bytes", DataType::Number(NumberDataType::UInt64)), ])) } + + fn children<'a>(&'a self) -> Box> + 'a> { + Box::new(std::iter::once(&self.input)) + } + + fn children_mut<'a>(&'a self) -> Box> + 'a> { + Box::new(std::iter::once(&mut self.input)) + } + + #[recursive::recursive] + fn try_find_single_data_source(&self) -> Option<&DataSourcePlan> { + self.input.try_find_single_data_source() + } + } diff --git a/src/query/sql/src/executor/physical_plans/physical_copy_into_table.rs b/src/query/sql/src/executor/physical_plans/physical_copy_into_table.rs index f1da86f95bdea..a3ea57acf6511 100644 --- a/src/query/sql/src/executor/physical_plans/physical_copy_into_table.rs +++ b/src/query/sql/src/executor/physical_plans/physical_copy_into_table.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use databend_common_catalog::plan::StageTableInfo; +use databend_common_catalog::plan::{DataSourcePlan, StageTableInfo}; use databend_common_exception::Result; use databend_common_expression::DataSchemaRef; use databend_common_expression::DataSchemaRefExt; @@ -25,12 +25,13 @@ use crate::executor::physical_plan::PhysicalPlan; use crate::plans::CopyIntoTableMode; use crate::plans::ValidationMode; use crate::ColumnBinding; -use crate::executor::IPhysicalPlan; +use crate::executor::{IPhysicalPlan, PhysicalPlanMeta}; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct CopyIntoTable { pub plan_id: u32, + meta: PhysicalPlanMeta, pub required_values_schema: DataSchemaRef, pub values_consts: Vec, pub required_source_schema: DataSchemaRef, @@ -46,17 +47,35 @@ pub struct CopyIntoTable { } impl IPhysicalPlan for CopyIntoTable { + fn get_meta(&self) -> &PhysicalPlanMeta { + &self.meta + } + + fn get_meta_mut(&mut self) -> &mut PhysicalPlanMeta { + &mut self.meta + } + fn output_schema(&self) -> Result { + Ok(DataSchemaRefExt::create(vec![])) + } + + fn children<'a>(&'a self) -> Box> + 'a> { + match &self.source { + CopyIntoTableSource::Query(v) => Box::new(std::iter::once(v)), + CopyIntoTableSource::Stage(v) => Box::new(std::iter::once(v)), + } + } + + fn children_mut<'a>(&'a self) -> Box> + 'a> { + match &mut self.source { + CopyIntoTableSource::Query(v) => Box::new(std::iter::once(v)), + CopyIntoTableSource::Stage(v) => Box::new(std::iter::once(v)), + } + } } #[derive(Clone, Debug, serde::Serialize, serde::Deserialize, EnumAsInner)] pub enum CopyIntoTableSource { - Query(Box), - Stage(Box), -} - -impl CopyIntoTable { - pub fn output_schema(&self) -> Result { - Ok(DataSchemaRefExt::create(vec![])) - } + Query(Box), + Stage(Box), } diff --git a/src/query/sql/src/executor/physical_plans/physical_distributed_insert_select.rs b/src/query/sql/src/executor/physical_plans/physical_distributed_insert_select.rs index 049c076bbf88f..001a4cee9e685 100644 --- a/src/query/sql/src/executor/physical_plans/physical_distributed_insert_select.rs +++ b/src/query/sql/src/executor/physical_plans/physical_distributed_insert_select.rs @@ -12,19 +12,22 @@ // See the License for the specific language governing permissions and // limitations under the License. +use databend_common_catalog::plan::DataSourcePlan; use databend_common_expression::DataSchemaRef; use databend_common_meta_app::schema::TableInfo; use databend_storages_common_table_meta::meta::TableMetaTimestamps; -use crate::executor::{IPhysicalPlan, PhysicalPlan}; +use crate::executor::{IPhysicalPlan, PhysicalPlan, PhysicalPlanMeta}; use crate::ColumnBinding; +use databend_common_exception::Result; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct DistributedInsertSelect { /// A unique id of operator in a `PhysicalPlan` tree. pub plan_id: u32, - pub input: Box, + meta: PhysicalPlanMeta, + pub input: Box, pub table_info: TableInfo, pub insert_schema: DataSchemaRef, pub select_schema: DataSchemaRef, @@ -34,5 +37,28 @@ pub struct DistributedInsertSelect { } impl IPhysicalPlan for DistributedInsertSelect { + fn get_meta(&self) -> &PhysicalPlanMeta { + &self.meta + } + fn get_meta_mut(&mut self) -> &mut PhysicalPlanMeta { + &mut self.meta + } + + fn output_schema(&self) -> Result { + Ok(DataSchemaRef::default()) + } + + fn children<'a>(&'a self) -> Box> + 'a> { + Box::new(std::iter::once(&self.input)) + } + + fn children_mut<'a>(&'a self) -> Box> + 'a> { + Box::new(std::iter::once(&mut self.input)) + } + + #[recursive::recursive] + fn try_find_single_data_source(&self) -> Option<&DataSourcePlan> { + self.input.try_find_single_data_source() + } } diff --git a/src/query/sql/src/executor/physical_plans/physical_eval_scalar.rs b/src/query/sql/src/executor/physical_plans/physical_eval_scalar.rs index 5e6f0bb91a9b7..22efac099b159 100644 --- a/src/query/sql/src/executor/physical_plans/physical_eval_scalar.rs +++ b/src/query/sql/src/executor/physical_plans/physical_eval_scalar.rs @@ -15,7 +15,7 @@ use std::collections::BTreeSet; use std::collections::HashSet; use std::sync::Arc; - +use databend_common_catalog::plan::DataSourcePlan; use databend_common_exception::Result; use databend_common_expression::ConstantFolder; use databend_common_expression::DataField; @@ -96,31 +96,10 @@ impl IPhysicalPlan for EvalScalar { fn children_mut<'a>(&'a self) -> Box> + 'a> { Box::new(std::iter::once(&mut self.input)) } -} -impl EvalScalar { - pub fn output_schema(&self) -> Result { - if self.exprs.is_empty() { - return self.input.output_schema(); - } - let input_schema = self.input.output_schema()?; - let mut fields = Vec::with_capacity(self.projections.len()); - for (i, field) in input_schema.fields().iter().enumerate() { - if self.projections.contains(&i) { - fields.push(field.clone()); - } - } - let input_column_nums = input_schema.num_fields(); - for (i, (expr, index)) in self.exprs.iter().enumerate() { - let i = i + input_column_nums; - if !self.projections.contains(&i) { - continue; - } - let name = index.to_string(); - let data_type = expr.as_expr(&BUILTIN_FUNCTIONS).data_type().clone(); - fields.push(DataField::new(&name, data_type)); - } - Ok(DataSchemaRefExt::create(fields)) + #[recursive::recursive] + fn try_find_single_data_source(&self) -> Option<&DataSourcePlan> { + self.input.try_find_single_data_source() } } diff --git a/src/query/sql/src/executor/physical_plans/physical_exchange.rs b/src/query/sql/src/executor/physical_plans/physical_exchange.rs index 151a6392d6d94..68a5152f066c8 100644 --- a/src/query/sql/src/executor/physical_plans/physical_exchange.rs +++ b/src/query/sql/src/executor/physical_plans/physical_exchange.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use databend_common_catalog::plan::DataSourcePlan; use databend_common_exception::Result; use databend_common_expression::ConstantFolder; use databend_common_expression::DataSchemaRef; @@ -19,7 +20,7 @@ use databend_common_expression::RemoteExpr; use databend_common_functions::BUILTIN_FUNCTIONS; use crate::executor::physical_plans::common::FragmentKind; -use crate::executor::{IPhysicalPlan, PhysicalPlan}; +use crate::executor::{IPhysicalPlan, PhysicalPlan, PhysicalPlanMeta}; use crate::executor::PhysicalPlanBuilder; use crate::optimizer::ir::SExpr; use crate::ColumnSet; @@ -30,7 +31,8 @@ pub struct Exchange { // A unique id of operator in a `PhysicalPlan` tree, only used for display. pub plan_id: u32, - pub input: Box, + meta: PhysicalPlanMeta, + pub input: Box, pub kind: FragmentKind, pub keys: Vec, pub ignore_exchange: bool, @@ -38,12 +40,29 @@ pub struct Exchange { } impl IPhysicalPlan for Exchange { + fn get_meta(&self) -> &PhysicalPlanMeta { + &self.meta + } -} + fn get_meta_mut(&mut self) -> &mut PhysicalPlanMeta { + &mut self.meta + } + + fn children<'a>(&'a self) -> Box> + 'a> { + Box::new(std::iter::once(&self.input)) + } + + fn children_mut<'a>(&'a self) -> Box> + 'a> { + Box::new(std::iter::once(&mut self.input)) + } + + #[recursive::recursive] + fn try_find_single_data_source(&self) -> Option<&DataSourcePlan> { + self.input.try_find_single_data_source() + } -impl Exchange { - pub fn output_schema(&self) -> Result { - self.input.output_schema() + fn is_distributed_plan(&self) -> bool { + true } } diff --git a/src/query/sql/src/executor/physical_plans/physical_exchange_sink.rs b/src/query/sql/src/executor/physical_plans/physical_exchange_sink.rs index 1c936bf177962..d0090caf3def7 100644 --- a/src/query/sql/src/executor/physical_plans/physical_exchange_sink.rs +++ b/src/query/sql/src/executor/physical_plans/physical_exchange_sink.rs @@ -12,18 +12,20 @@ // See the License for the specific language governing permissions and // limitations under the License. +use databend_common_catalog::plan::DataSourcePlan; use databend_common_exception::Result; use databend_common_expression::DataSchemaRef; use databend_common_expression::RemoteExpr; use crate::executor::physical_plans::common::FragmentKind; -use crate::executor::{IPhysicalPlan, PhysicalPlan}; +use crate::executor::{IPhysicalPlan, PhysicalPlan, PhysicalPlanMeta}; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct ExchangeSink { // A unique id of operator in a `PhysicalPlan` tree, only used for display. pub plan_id: u32, - pub input: Box, + meta: PhysicalPlanMeta, + pub input: Box, // Input schema of exchanged data pub schema: DataSchemaRef, pub kind: FragmentKind, @@ -39,11 +41,32 @@ pub struct ExchangeSink { } impl IPhysicalPlan for ExchangeSink { + fn get_meta(&self) -> &PhysicalPlanMeta { + &self.meta + } -} + fn get_meta_mut(&mut self) -> &mut PhysicalPlanMeta { + &mut self.meta + } -impl ExchangeSink { - pub fn output_schema(&self) -> Result { + fn output_schema(&self) -> Result { Ok(self.schema.clone()) } + + fn children<'a>(&'a self) -> Box> + 'a> { + Box::new(std::iter::once(&self.input)) + } + + fn children_mut<'a>(&'a self) -> Box> + 'a> { + Box::new(std::iter::once(&mut self.input)) + } + + #[recursive::recursive] + fn try_find_single_data_source(&self) -> Option<&DataSourcePlan> { + self.input.try_find_single_data_source() + } + + fn is_distributed_plan(&self) -> bool { + true + } } diff --git a/src/query/sql/src/executor/physical_plans/physical_exchange_source.rs b/src/query/sql/src/executor/physical_plans/physical_exchange_source.rs index fabb0f1f458c5..2b56dfa1e967b 100644 --- a/src/query/sql/src/executor/physical_plans/physical_exchange_source.rs +++ b/src/query/sql/src/executor/physical_plans/physical_exchange_source.rs @@ -12,14 +12,16 @@ // See the License for the specific language governing permissions and // limitations under the License. +use databend_common_catalog::plan::DataSourcePlan; use databend_common_exception::Result; use databend_common_expression::DataSchemaRef; -use crate::executor::IPhysicalPlan; +use crate::executor::{IPhysicalPlan, PhysicalPlanMeta}; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct ExchangeSource { // A unique id of operator in a `PhysicalPlan` tree, only used for display. pub plan_id: u32, + meta: PhysicalPlanMeta, // Output schema of exchanged data pub schema: DataSchemaRef, @@ -30,11 +32,19 @@ pub struct ExchangeSource { } impl IPhysicalPlan for ExchangeSource { + fn get_meta(&self) -> &PhysicalPlanMeta { + &self.meta + } -} + fn get_meta_mut(&mut self) -> &mut PhysicalPlanMeta { + &mut self.meta + } -impl ExchangeSource { - pub fn output_schema(&self) -> Result { + fn output_schema(&self) -> Result { Ok(self.schema.clone()) } + + fn is_distributed_plan(&self) -> bool { + true + } } diff --git a/src/query/sql/src/executor/physical_plans/physical_expression_scan.rs b/src/query/sql/src/executor/physical_plans/physical_expression_scan.rs index aabb23af935aa..5d9026bef5e49 100644 --- a/src/query/sql/src/executor/physical_plans/physical_expression_scan.rs +++ b/src/query/sql/src/executor/physical_plans/physical_expression_scan.rs @@ -18,7 +18,7 @@ use databend_common_expression::DataSchemaRef; use databend_common_expression::RemoteExpr; use databend_common_functions::BUILTIN_FUNCTIONS; -use crate::executor::{IPhysicalPlan, PhysicalPlan}; +use crate::executor::{IPhysicalPlan, PhysicalPlan, PhysicalPlanMeta}; use crate::executor::PhysicalPlanBuilder; use crate::optimizer::ir::SExpr; use crate::ColumnSet; @@ -28,19 +28,32 @@ use crate::TypeCheck; pub struct ExpressionScan { // A unique id of operator in a `PhysicalPlan` tree, only used for display. pub plan_id: u32, + meta: PhysicalPlanMeta, pub values: Vec>, - pub input: Box, + pub input: Box, pub output_schema: DataSchemaRef, } impl IPhysicalPlan for ExpressionScan { + fn get_meta(&self) -> &PhysicalPlanMeta { + &self.meta + } -} + fn get_meta_mut(&mut self) -> &mut PhysicalPlanMeta { + &mut self.meta + } -impl ExpressionScan { - pub fn output_schema(&self) -> Result { + fn output_schema(&self) -> Result { Ok(self.output_schema.clone()) } + + fn children<'a>(&'a self) -> Box> + 'a> { + Box::new(std::iter::once(&self.input)) + } + + fn children_mut<'a>(&'a self) -> Box> + 'a> { + Box::new(std::iter::once(&mut self.input)) + } } impl PhysicalPlanBuilder { diff --git a/src/query/sql/src/executor/physical_plans/physical_filter.rs b/src/query/sql/src/executor/physical_plans/physical_filter.rs index 8a8ce1fae89be..b0325b9b12292 100644 --- a/src/query/sql/src/executor/physical_plans/physical_filter.rs +++ b/src/query/sql/src/executor/physical_plans/physical_filter.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use databend_common_catalog::plan::DataSourcePlan; use databend_common_exception::Result; use databend_common_expression::ConstantFolder; use databend_common_expression::DataSchemaRef; @@ -68,18 +69,10 @@ impl IPhysicalPlan for Filter { } Ok(DataSchemaRefExt::create(fields)) } -} -impl Filter { - pub fn output_schema(&self) -> Result { - let input_schema = self.input.output_schema()?; - let mut fields = Vec::with_capacity(self.projections.len()); - for (i, field) in input_schema.fields().iter().enumerate() { - if self.projections.contains(&i) { - fields.push(field.clone()); - } - } - Ok(DataSchemaRefExt::create(fields)) + #[recursive::recursive] + fn try_find_single_data_source(&self) -> Option<&DataSourcePlan> { + self.input.try_find_single_data_source() } } diff --git a/src/query/sql/src/executor/physical_plans/physical_hash_join.rs b/src/query/sql/src/executor/physical_plans/physical_hash_join.rs index dc7004b1e372d..cd57535f66577 100644 --- a/src/query/sql/src/executor/physical_plans/physical_hash_join.rs +++ b/src/query/sql/src/executor/physical_plans/physical_hash_join.rs @@ -77,8 +77,8 @@ pub struct HashJoin { pub probe_projections: ColumnSet, pub build_projections: ColumnSet, - pub build: Box, - pub probe: Box, + pub build: Box, + pub probe: Box, pub build_keys: Vec, pub probe_keys: Vec, pub is_null_equal: Vec, @@ -123,13 +123,11 @@ impl IPhysicalPlan for HashJoin { } fn children<'a>(&'a self) -> Box> + 'a> { - todo!() + Box::new(std::iter::once(&self.probe).chain(std::iter::once(&self.build))) } -} -impl HashJoin { - pub fn output_schema(&self) -> Result { - Ok(self.output_schema.clone()) + fn children_mut<'a>(&'a self) -> Box> + 'a> { + Box::new(std::iter::once(&mut self.probe).chain(std::iter::once(&mut self.build))) } } diff --git a/src/query/sql/src/executor/physical_plans/physical_limit.rs b/src/query/sql/src/executor/physical_plans/physical_limit.rs index ea0f523f66fc1..4ed509b86084f 100644 --- a/src/query/sql/src/executor/physical_plans/physical_limit.rs +++ b/src/query/sql/src/executor/physical_plans/physical_limit.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use databend_common_catalog::plan::DataSourcePlan; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_expression::DataField; @@ -55,11 +56,10 @@ impl IPhysicalPlan for Limit { fn children_mut<'a>(&'a self) -> Box> + 'a> { Box::new(std::iter::once(&mut self.input)) } -} -impl Limit { - pub fn output_schema(&self) -> Result { - self.input.output_schema() + #[recursive::recursive] + fn try_find_single_data_source(&self) -> Option<&DataSourcePlan> { + self.input.try_find_single_data_source() } } diff --git a/src/query/sql/src/executor/physical_plans/physical_multi_table_insert.rs b/src/query/sql/src/executor/physical_plans/physical_multi_table_insert.rs index 8f4a08640643a..f644ebf57b01f 100644 --- a/src/query/sql/src/executor/physical_plans/physical_multi_table_insert.rs +++ b/src/query/sql/src/executor/physical_plans/physical_multi_table_insert.rs @@ -23,25 +23,59 @@ use databend_common_meta_app::schema::TableInfo; use databend_common_meta_app::schema::UpdateStreamMetaReq; use databend_storages_common_table_meta::meta::TableMetaTimestamps; -use crate::executor::{IPhysicalPlan, PhysicalPlan}; +use crate::executor::{IPhysicalPlan, PhysicalPlan, PhysicalPlanMeta}; use crate::ColumnSet; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct Duplicate { pub plan_id: u32, - pub input: Box, + meta: PhysicalPlanMeta, + pub input: Box, pub n: usize, } -impl IPhysicalPlan for Duplicate {} +impl IPhysicalPlan for Duplicate { + fn get_meta(&self) -> &PhysicalPlanMeta { + &self.meta + } + + fn get_meta_mut(&mut self) -> &mut PhysicalPlanMeta { + &mut self.meta + } + + fn children<'a>(&'a self) -> Box> + 'a> { + Box::new(std::iter::once(&self.input)) + } + + fn children_mut<'a>(&'a self) -> Box> + 'a> { + Box::new(std::iter::once(&mut self.input)) + } +} #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct Shuffle { pub plan_id: u32, - pub input: Box, + meta: PhysicalPlanMeta, + pub input: Box, pub strategy: ShuffleStrategy, } -impl IPhysicalPlan for Shuffle {} +impl IPhysicalPlan for Shuffle { + fn get_meta(&self) -> &PhysicalPlanMeta { + &self.meta + } + + fn get_meta_mut(&mut self) -> &mut PhysicalPlanMeta { + &mut self.meta + } + + fn children<'a>(&'a self) -> Box> + 'a> { + Box::new(std::iter::once(&self.input)) + } + + fn children_mut<'a>(&'a self) -> Box> + 'a> { + Box::new(std::iter::once(&mut self.input)) + } +} #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub enum ShuffleStrategy { @@ -73,20 +107,54 @@ impl ShuffleStrategy { #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct ChunkFilter { pub plan_id: u32, - pub input: Box, + meta: PhysicalPlanMeta, + pub input: Box, pub predicates: Vec>, } -impl IPhysicalPlan for ChunkFilter {} +impl IPhysicalPlan for ChunkFilter { + fn get_meta(&self) -> &PhysicalPlanMeta { + &self.meta + } + + fn get_meta_mut(&mut self) -> &mut PhysicalPlanMeta { + &mut self.meta + } + + fn children<'a>(&'a self) -> Box> + 'a> { + Box::new(std::iter::once(&self.input)) + } + + fn children_mut<'a>(&'a self) -> Box> + 'a> { + Box::new(std::iter::once(&mut self.input)) + } +} #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct ChunkEvalScalar { pub plan_id: u32, - pub input: Box, + meta: PhysicalPlanMeta, + pub input: Box, pub eval_scalars: Vec>, } -impl IPhysicalPlan for ChunkEvalScalar {} +impl IPhysicalPlan for ChunkEvalScalar { + fn get_meta(&self) -> &PhysicalPlanMeta { + &self.meta + } + + fn get_meta_mut(&mut self) -> &mut PhysicalPlanMeta { + &mut self.meta + } + + fn children<'a>(&'a self) -> Box> + 'a> { + Box::new(std::iter::once(&self.input)) + } + + fn children_mut<'a>(&'a self) -> Box> + 'a> { + Box::new(std::iter::once(&mut self.input)) + } +} #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct MultiInsertEvalScalar { @@ -97,11 +165,28 @@ pub struct MultiInsertEvalScalar { #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct ChunkCastSchema { pub plan_id: u32, - pub input: Box, + meta: PhysicalPlanMeta, + pub input: Box, pub cast_schemas: Vec>, } -impl IPhysicalPlan for ChunkCastSchema {} +impl IPhysicalPlan for ChunkCastSchema { + fn get_meta(&self) -> &PhysicalPlanMeta { + &self.meta + } + + fn get_meta_mut(&mut self) -> &mut PhysicalPlanMeta { + &mut self.meta + } + + fn children<'a>(&'a self) -> Box> + 'a> { + Box::new(std::iter::once(&self.input)) + } + + fn children_mut<'a>(&'a self) -> Box> + 'a> { + Box::new(std::iter::once(&mut self.input)) + } +} #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct CastSchema { @@ -112,11 +197,28 @@ pub struct CastSchema { #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct ChunkFillAndReorder { pub plan_id: u32, - pub input: Box, + meta: PhysicalPlanMeta, + pub input: Box, pub fill_and_reorders: Vec>, } -impl IPhysicalPlan for ChunkFillAndReorder {} +impl IPhysicalPlan for ChunkFillAndReorder { + fn get_meta(&self) -> &PhysicalPlanMeta { + &self.meta + } + + fn get_meta_mut(&mut self) -> &mut PhysicalPlanMeta { + &mut self.meta + } + + fn children<'a>(&'a self) -> Box> + 'a> { + Box::new(std::iter::once(&self.input)) + } + + fn children_mut<'a>(&'a self) -> Box> + 'a> { + Box::new(std::iter::once(&mut self.input)) + } +} #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct FillAndReorder { @@ -127,11 +229,28 @@ pub struct FillAndReorder { #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct ChunkAppendData { pub plan_id: u32, - pub input: Box, + meta: PhysicalPlanMeta, + pub input: Box, pub target_tables: Vec, } -impl IPhysicalPlan for ChunkAppendData {} +impl IPhysicalPlan for ChunkAppendData { + fn get_meta(&self) -> &PhysicalPlanMeta { + &self.meta + } + + fn get_meta_mut(&mut self) -> &mut PhysicalPlanMeta { + &mut self.meta + } + + fn children<'a>(&'a self) -> Box> + 'a> { + Box::new(std::iter::once(&self.input)) + } + + fn children_mut<'a>(&'a self) -> Box> + 'a> { + Box::new(std::iter::once(&mut self.input)) + } +} #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct SerializableTable { @@ -143,16 +262,34 @@ pub struct SerializableTable { #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct ChunkMerge { pub plan_id: u32, - pub input: Box, + meta: PhysicalPlanMeta, + pub input: Box, pub group_ids: Vec, } -impl IPhysicalPlan for ChunkMerge {} +impl IPhysicalPlan for ChunkMerge { + fn get_meta(&self) -> &PhysicalPlanMeta { + &self.meta + } + + fn get_meta_mut(&mut self) -> &mut PhysicalPlanMeta { + &mut self.meta + } + + fn children<'a>(&'a self) -> Box> + 'a> { + Box::new(std::iter::once(&self.input)) + } + + fn children_mut<'a>(&'a self) -> Box> + 'a> { + Box::new(std::iter::once(&mut self.input)) + } +} #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct ChunkCommitInsert { pub plan_id: u32, - pub input: Box, + meta: PhysicalPlanMeta, + pub input: Box, pub update_stream_meta: Vec, pub overwrite: bool, pub deduplicated_label: Option, @@ -160,5 +297,19 @@ pub struct ChunkCommitInsert { } impl IPhysicalPlan for ChunkCommitInsert { + fn get_meta(&self) -> &PhysicalPlanMeta { + &self.meta + } + + fn get_meta_mut(&mut self) -> &mut PhysicalPlanMeta { + &mut self.meta + } + + fn children<'a>(&'a self) -> Box> + 'a> { + Box::new(std::iter::once(&self.input)) + } + fn children_mut<'a>(&'a self) -> Box> + 'a> { + Box::new(std::iter::once(&mut self.input)) + } } diff --git a/src/query/sql/src/executor/physical_plans/physical_mutation.rs b/src/query/sql/src/executor/physical_plans/physical_mutation.rs index 78deb7450b46b..31f949ee73b80 100644 --- a/src/query/sql/src/executor/physical_plans/physical_mutation.rs +++ b/src/query/sql/src/executor/physical_plans/physical_mutation.rs @@ -15,7 +15,7 @@ use std::collections::HashMap; use std::sync::Arc; -use databend_common_catalog::plan::NUM_ROW_ID_PREFIX_BITS; +use databend_common_catalog::plan::{DataSourcePlan, NUM_ROW_ID_PREFIX_BITS}; use databend_common_catalog::table_context::TableContext; use databend_common_exception::ErrorCode; use databend_common_exception::Result; @@ -57,7 +57,7 @@ use crate::executor::physical_plans::MutationManipulate; use crate::executor::physical_plans::MutationOrganize; use crate::executor::physical_plans::MutationSplit; use crate::executor::physical_plans::RowFetch; -use crate::executor::{IPhysicalPlan, PhysicalPlanBuilder}; +use crate::executor::{IPhysicalPlan, PhysicalPlanBuilder, PhysicalPlanMeta}; use crate::optimizer::ir::SExpr; use crate::parse_computed_expr; use crate::plans::BoundColumnRef; @@ -82,7 +82,8 @@ pub type MatchExpr = Vec<(Option, Option, + meta: PhysicalPlanMeta, + pub input: Box, pub table_info: TableInfo, // (DataSchemaRef, Option, Vec,Vec) => (source_schema, condition, value_exprs) pub unmatched: Vec<(DataSchemaRef, Option, Vec)>, @@ -96,7 +97,25 @@ pub struct Mutation { } impl IPhysicalPlan for Mutation { + fn get_meta(&self) -> &PhysicalPlanMeta { + &self.meta + } + + fn get_meta_mut(&mut self) -> &mut PhysicalPlanMeta { + &mut self.meta + } + + fn output_schema(&self) -> Result { + Ok(DataSchemaRef::default()) + } + + fn children<'a>(&'a self) -> Box> + 'a> { + Box::new(std::iter::once(&self.input)) + } + fn children_mut<'a>(&'a self) -> Box> + 'a> { + Box::new(std::iter::once(&mut self.input)) + } } impl PhysicalPlanBuilder { diff --git a/src/query/sql/src/executor/physical_plans/physical_mutation_into_organize.rs b/src/query/sql/src/executor/physical_plans/physical_mutation_into_organize.rs index 62c555473b0ae..de83e71a75677 100644 --- a/src/query/sql/src/executor/physical_plans/physical_mutation_into_organize.rs +++ b/src/query/sql/src/executor/physical_plans/physical_mutation_into_organize.rs @@ -16,22 +16,31 @@ use databend_common_exception::Result; use databend_common_expression::DataSchemaRef; use crate::binder::MutationStrategy; -use crate::executor::IPhysicalPlan; +use crate::executor::{IPhysicalPlan, PhysicalPlanMeta}; use crate::executor::physical_plan::PhysicalPlan; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct MutationOrganize { pub plan_id: u32, - pub input: Box, + meta: PhysicalPlanMeta, + pub input: Box, pub strategy: MutationStrategy, } impl IPhysicalPlan for MutationOrganize { + fn get_meta(&self) -> &PhysicalPlanMeta { + &self.meta + } -} + fn get_meta_mut(&mut self) -> &mut PhysicalPlanMeta { + &mut self.meta + } + + fn children<'a>(&'a self) -> Box> + 'a> { + Box::new(std::iter::once(&self.input)) + } -impl MutationOrganize { - pub fn output_schema(&self) -> Result { - self.input.output_schema() + fn children_mut<'a>(&'a self) -> Box> + 'a> { + Box::new(std::iter::once(&mut self.input)) } } diff --git a/src/query/sql/src/executor/physical_plans/physical_mutation_into_split.rs b/src/query/sql/src/executor/physical_plans/physical_mutation_into_split.rs index ace7afb551215..ebb0704386aa8 100644 --- a/src/query/sql/src/executor/physical_plans/physical_mutation_into_split.rs +++ b/src/query/sql/src/executor/physical_plans/physical_mutation_into_split.rs @@ -14,23 +14,32 @@ use databend_common_exception::Result; use databend_common_expression::DataSchemaRef; -use crate::executor::IPhysicalPlan; +use crate::executor::{IPhysicalPlan, PhysicalPlanMeta}; use crate::executor::physical_plan::PhysicalPlan; use crate::IndexType; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct MutationSplit { pub plan_id: u32, - pub input: Box, + meta: PhysicalPlanMeta, + pub input: Box, pub split_index: IndexType, } impl IPhysicalPlan for MutationSplit { + fn get_meta(&self) -> &PhysicalPlanMeta { + &self.meta + } -} + fn get_meta_mut(&mut self) -> &mut PhysicalPlanMeta { + &mut self.meta + } + + fn children<'a>(&'a self) -> Box> + 'a> { + Box::new(std::iter::once(&self.input)) + } -impl MutationSplit { - pub fn output_schema(&self) -> Result { - self.input.output_schema() + fn children_mut<'a>(&'a self) -> Box> + 'a> { + Box::new(std::iter::once(&mut self.input)) } } diff --git a/src/query/sql/src/executor/physical_plans/physical_mutation_manipulate.rs b/src/query/sql/src/executor/physical_plans/physical_mutation_manipulate.rs index 38b21206c90e9..cf3194df9781d 100644 --- a/src/query/sql/src/executor/physical_plans/physical_mutation_manipulate.rs +++ b/src/query/sql/src/executor/physical_plans/physical_mutation_manipulate.rs @@ -21,7 +21,7 @@ use databend_common_expression::RemoteExpr; use databend_common_meta_app::schema::TableInfo; use crate::binder::MutationStrategy; -use crate::executor::IPhysicalPlan; +use crate::executor::{IPhysicalPlan, PhysicalPlanMeta}; use crate::executor::physical_plan::PhysicalPlan; pub type MatchExpr = Vec<(Option, Option>)>; @@ -29,7 +29,8 @@ pub type MatchExpr = Vec<(Option, Option, + meta: PhysicalPlanMeta, + pub input: Box, pub table_info: TableInfo, // (DataSchemaRef, Option, Vec,Vec) => (source_schema, condition, value_exprs) pub unmatched: Vec<(DataSchemaRef, Option, Vec)>, @@ -45,11 +46,19 @@ pub struct MutationManipulate { } impl IPhysicalPlan for MutationManipulate { + fn get_meta(&self) -> &PhysicalPlanMeta { + &self.meta + } -} + fn get_meta_mut(&mut self) -> &mut PhysicalPlanMeta { + &mut self.meta + } + + fn children<'a>(&'a self) -> Box> + 'a> { + Box::new(std::iter::once(&self.input)) + } -impl MutationManipulate { - pub fn output_schema(&self) -> Result { - self.input.output_schema() + fn children_mut<'a>(&'a self) -> Box> + 'a> { + Box::new(std::iter::once(&mut self.input)) } } diff --git a/src/query/sql/src/executor/physical_plans/physical_mutation_source.rs b/src/query/sql/src/executor/physical_plans/physical_mutation_source.rs index e7ebae33d3c19..c0180e7bfa567 100644 --- a/src/query/sql/src/executor/physical_plans/physical_mutation_source.rs +++ b/src/query/sql/src/executor/physical_plans/physical_mutation_source.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use databend_common_catalog::plan::Filters; +use databend_common_catalog::plan::{DataSourcePlan, Filters}; use databend_common_catalog::plan::PartStatistics; use databend_common_catalog::plan::Partitions; use databend_common_exception::Result; @@ -27,7 +27,7 @@ use databend_common_functions::BUILTIN_FUNCTIONS; use databend_common_meta_app::schema::TableInfo; use crate::binder::MutationType; -use crate::executor::{cast_expr_to_non_null_boolean, IPhysicalPlan}; +use crate::executor::{cast_expr_to_non_null_boolean, IPhysicalPlan, PhysicalPlanMeta}; use crate::executor::PhysicalPlan; use crate::executor::PhysicalPlanBuilder; use crate::ColumnSet; @@ -38,6 +38,7 @@ use crate::ScalarExpr; pub struct MutationSource { // A unique id of operator in a `PhysicalPlan` tree, only used for display. pub plan_id: u32, + meta: PhysicalPlanMeta, pub table_index: IndexType, pub table_info: TableInfo, pub filters: Option, @@ -51,13 +52,21 @@ pub struct MutationSource { } impl IPhysicalPlan for MutationSource { + fn get_meta(&self) -> &PhysicalPlanMeta { + &self.meta + } -} + fn get_meta_mut(&mut self) -> &mut PhysicalPlanMeta { + &mut self.meta + } -impl MutationSource { - pub fn output_schema(&self) -> Result { + fn output_schema(&self) -> Result { Ok(self.output_schema.clone()) } + + fn try_find_mutation_source(&self) -> Option { + Some(self.clone()) + } } impl PhysicalPlanBuilder { diff --git a/src/query/sql/src/executor/physical_plans/physical_project_set.rs b/src/query/sql/src/executor/physical_plans/physical_project_set.rs index 740341d319fad..630a3a80b33a4 100644 --- a/src/query/sql/src/executor/physical_plans/physical_project_set.rs +++ b/src/query/sql/src/executor/physical_plans/physical_project_set.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use databend_common_catalog::plan::DataSourcePlan; use databend_common_exception::Result; use databend_common_expression::ConstantFolder; use databend_common_expression::DataField; @@ -74,24 +75,10 @@ impl IPhysicalPlan for ProjectSet { fn children_mut<'a>(&'a self) -> Box> + 'a> { Box::new(std::iter::once(&mut self.input)) } -} -impl ProjectSet { - pub fn output_schema(&self) -> Result { - let input_schema = self.input.output_schema()?; - let mut fields = Vec::with_capacity(input_schema.num_fields() + self.srf_exprs.len()); - for (i, field) in input_schema.fields().iter().enumerate() { - if self.projections.contains(&i) { - fields.push(field.clone()); - } - } - fields.extend(self.srf_exprs.iter().map(|(srf, index)| { - DataField::new( - &index.to_string(), - srf.as_expr(&BUILTIN_FUNCTIONS).data_type().clone(), - ) - })); - Ok(DataSchemaRefExt::create(fields)) + #[recursive::recursive] + fn try_find_single_data_source(&self) -> Option<&DataSourcePlan> { + self.input.try_find_single_data_source() } } diff --git a/src/query/sql/src/executor/physical_plans/physical_r_cte_scan.rs b/src/query/sql/src/executor/physical_plans/physical_r_cte_scan.rs index 48a16c37e7679..929946404a18a 100644 --- a/src/query/sql/src/executor/physical_plans/physical_r_cte_scan.rs +++ b/src/query/sql/src/executor/physical_plans/physical_r_cte_scan.rs @@ -19,24 +19,29 @@ use databend_common_expression::DataSchemaRef; use databend_common_expression::DataSchemaRefExt; use crate::executor::explain::PlanStatsInfo; -use crate::executor::{IPhysicalPlan, PhysicalPlan}; +use crate::executor::{IPhysicalPlan, PhysicalPlan, PhysicalPlanMeta}; use crate::executor::PhysicalPlanBuilder; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct RecursiveCteScan { // A unique id of operator in a `PhysicalPlan` tree, only used for display. pub plan_id: u32, + meta: PhysicalPlanMeta, pub output_schema: DataSchemaRef, pub table_name: String, pub stat: PlanStatsInfo, } impl IPhysicalPlan for RecursiveCteScan { + fn get_meta(&self) -> &PhysicalPlanMeta { + &self.meta + } -} + fn get_meta_mut(&mut self) -> &mut PhysicalPlanMeta { + &mut self.meta + } -impl RecursiveCteScan { - pub fn output_schema(&self) -> Result { + fn output_schema(&self) -> Result { Ok(self.output_schema.clone()) } } diff --git a/src/query/sql/src/executor/physical_plans/physical_range_join.rs b/src/query/sql/src/executor/physical_plans/physical_range_join.rs index b6ca2e654ae84..1b1bd4a5b9128 100644 --- a/src/query/sql/src/executor/physical_plans/physical_range_join.rs +++ b/src/query/sql/src/executor/physical_plans/physical_range_join.rs @@ -26,7 +26,7 @@ use databend_common_functions::BUILTIN_FUNCTIONS; use crate::binder::wrap_cast; use crate::binder::JoinPredicate; use crate::executor::explain::PlanStatsInfo; -use crate::executor::{IPhysicalPlan, PhysicalPlan}; +use crate::executor::{IPhysicalPlan, PhysicalPlan, PhysicalPlanMeta}; use crate::executor::PhysicalPlanBuilder; use crate::optimizer::ir::RelExpr; use crate::optimizer::ir::RelationalProperty; @@ -41,6 +41,7 @@ use crate::TypeCheck; pub struct RangeJoin { // A unique id of operator in a `PhysicalPlan` tree, only used for display. pub plan_id: u32, + meta: PhysicalPlanMeta, pub left: Box, pub right: Box, // The first two conditions: (>, >=, <, <=) @@ -58,13 +59,25 @@ pub struct RangeJoin { } impl IPhysicalPlan for RangeJoin { + fn get_meta(&self) -> &PhysicalPlanMeta { + &self.meta + } -} + fn get_meta_mut(&mut self) -> &mut PhysicalPlanMeta { + &mut self.meta + } -impl RangeJoin { - pub fn output_schema(&self) -> Result { + fn output_schema(&self) -> Result { Ok(self.output_schema.clone()) } + + fn children<'a>(&'a self) -> Box> + 'a> { + Box::new(std::iter::once(&self.left).chain(std::iter::once(&self.right))) + } + + fn children_mut<'a>(&'a self) -> Box> + 'a> { + Box::new(std::iter::once(&mut self.left).chain(std::iter::once(&mut self.right))) + } } #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] @@ -176,10 +189,10 @@ fn resolve_range_condition( let common_type = common_super_type(arg1_data_type.clone(), arg2_data_type.clone(), cast_rules) .ok_or_else(|| { - ErrorCode::IllegalDataType(format!( - "Cannot find common type for {arg1_data_type} and {arg2_data_type}" - )) - })?; + ErrorCode::IllegalDataType(format!( + "Cannot find common type for {arg1_data_type} and {arg2_data_type}" + )) + })?; arg1 = wrap_cast(&arg1, &common_type); arg2 = wrap_cast(&arg2, &common_type); }; diff --git a/src/query/sql/src/executor/physical_plans/physical_recluster.rs b/src/query/sql/src/executor/physical_plans/physical_recluster.rs index 592bc74a142c6..b510b4f295cbd 100644 --- a/src/query/sql/src/executor/physical_plans/physical_recluster.rs +++ b/src/query/sql/src/executor/physical_plans/physical_recluster.rs @@ -12,28 +12,38 @@ // See the License for the specific language governing permissions and // limitations under the License. -use databend_common_catalog::plan::ReclusterTask; +use databend_common_catalog::plan::{DataSourcePlan, ReclusterTask}; +use databend_common_expression::DataSchemaRef; use databend_common_meta_app::schema::TableInfo; use databend_storages_common_table_meta::meta::TableMetaTimestamps; -use crate::executor::{IPhysicalPlan, PhysicalPlan}; +use crate::executor::{IPhysicalPlan, PhysicalPlan, PhysicalPlanMeta}; +use databend_common_exception::Result; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct Recluster { pub plan_id: u32, + meta: PhysicalPlanMeta, pub tasks: Vec, pub table_info: TableInfo, pub table_meta_timestamps: TableMetaTimestamps, } impl IPhysicalPlan for Recluster { + fn get_meta(&self) -> &PhysicalPlanMeta { + &self.meta + } + fn get_meta_mut(&mut self) -> &mut PhysicalPlanMeta { + &mut self.meta + } } #[derive(serde::Serialize, serde::Deserialize, Clone, Debug)] pub struct HilbertPartition { pub plan_id: u32, - pub input: Box, + meta: PhysicalPlanMeta, + pub input: Box, pub table_info: TableInfo, pub num_partitions: usize, pub table_meta_timestamps: TableMetaTimestamps, @@ -41,5 +51,23 @@ pub struct HilbertPartition { } impl IPhysicalPlan for HilbertPartition { + fn get_meta(&self) -> &PhysicalPlanMeta { + &self.meta + } + fn get_meta_mut(&mut self) -> &mut PhysicalPlanMeta { + &mut self.meta + } + + fn output_schema(&self) -> Result { + Ok(DataSchemaRef::default()) + } + + fn children<'a>(&'a self) -> Box> + 'a> { + Box::new(std::iter::once(&self.input)) + } + + fn children_mut<'a>(&'a self) -> Box> + 'a> { + Box::new(std::iter::once(&mut self.input)) + } } diff --git a/src/query/sql/src/executor/physical_plans/physical_replace_async_source.rs b/src/query/sql/src/executor/physical_plans/physical_replace_async_source.rs index 872896518ec96..570a0d92ce668 100644 --- a/src/query/sql/src/executor/physical_plans/physical_replace_async_source.rs +++ b/src/query/sql/src/executor/physical_plans/physical_replace_async_source.rs @@ -12,17 +12,25 @@ // See the License for the specific language governing permissions and // limitations under the License. +use databend_common_catalog::plan::DataSourcePlan; use databend_common_expression::DataSchemaRef; -use crate::executor::IPhysicalPlan; +use crate::executor::{IPhysicalPlan, PhysicalPlanMeta}; use crate::plans::InsertValue; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct ReplaceAsyncSourcer { pub plan_id: u32, + meta: PhysicalPlanMeta, pub schema: DataSchemaRef, pub source: InsertValue, } impl IPhysicalPlan for ReplaceAsyncSourcer { + fn get_meta(&self) -> &PhysicalPlanMeta { + &self.meta + } + fn get_meta_mut(&mut self) -> &mut PhysicalPlanMeta { + &mut self.meta + } } diff --git a/src/query/sql/src/executor/physical_plans/physical_replace_deduplicate.rs b/src/query/sql/src/executor/physical_plans/physical_replace_deduplicate.rs index 3f7c5617d7cf4..7d91fff666cd1 100644 --- a/src/query/sql/src/executor/physical_plans/physical_replace_deduplicate.rs +++ b/src/query/sql/src/executor/physical_plans/physical_replace_deduplicate.rs @@ -13,7 +13,7 @@ // limitations under the License. use std::collections::HashMap; - +use databend_common_catalog::plan::DataSourcePlan; use databend_common_expression::ColumnId; use databend_common_expression::DataSchemaRef; use databend_common_expression::FieldIndex; @@ -23,13 +23,15 @@ use databend_common_meta_app::schema::TableInfo; use databend_storages_common_table_meta::meta::ColumnStatistics; use crate::executor::physical_plans::common::OnConflictField; -use crate::executor::{IPhysicalPlan, PhysicalPlan}; +use crate::executor::{IPhysicalPlan, PhysicalPlan, PhysicalPlanMeta}; use crate::ColumnBinding; +use databend_common_exception::Result; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct ReplaceDeduplicate { pub plan_id: u32, - pub input: Box, + meta: PhysicalPlanMeta, + pub input: Box, pub on_conflicts: Vec, pub bloom_filter_column_indexes: Vec, pub table_is_empty: bool, @@ -42,7 +44,25 @@ pub struct ReplaceDeduplicate { } impl IPhysicalPlan for ReplaceDeduplicate { + fn get_meta(&self) -> &PhysicalPlanMeta { + &self.meta + } + + fn get_meta_mut(&mut self) -> &mut PhysicalPlanMeta { + &mut self.meta + } + + fn output_schema(&self) -> Result { + Ok(DataSchemaRef::default()) + } + + fn children<'a>(&'a self) -> Box> + 'a> { + Box::new(std::iter::once(&self.input)) + } + fn children_mut<'a>(&'a self) -> Box> + 'a> { + Box::new(std::iter::once(&mut self.input)) + } } #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] diff --git a/src/query/sql/src/executor/physical_plans/physical_replace_into.rs b/src/query/sql/src/executor/physical_plans/physical_replace_into.rs index e58c4cd97fd6b..bd2815dfbb745 100644 --- a/src/query/sql/src/executor/physical_plans/physical_replace_into.rs +++ b/src/query/sql/src/executor/physical_plans/physical_replace_into.rs @@ -12,7 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -use databend_common_expression::BlockThresholds; +use databend_common_catalog::plan::DataSourcePlan; +use databend_common_expression::{BlockThresholds, DataSchemaRef}; use databend_common_expression::FieldIndex; use databend_common_meta_app::schema::TableInfo; use databend_storages_common_table_meta::meta::BlockSlotDescription; @@ -20,14 +21,16 @@ use databend_storages_common_table_meta::meta::Location; use databend_storages_common_table_meta::meta::TableMetaTimestamps; use crate::executor::physical_plans::common::OnConflictField; -use crate::executor::{IPhysicalPlan, PhysicalPlan}; +use crate::executor::{IPhysicalPlan, PhysicalPlan, PhysicalPlanMeta}; +use databend_common_exception::Result; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct ReplaceInto { /// A unique id of operator in a `PhysicalPlan` tree. pub plan_id: u32, - pub input: Box, + meta: PhysicalPlanMeta, + pub input: Box, pub block_thresholds: BlockThresholds, pub table_info: TableInfo, pub on_conflicts: Vec, @@ -39,5 +42,23 @@ pub struct ReplaceInto { } impl IPhysicalPlan for ReplaceInto { + fn get_meta(&self) -> &PhysicalPlanMeta { + &self.meta + } + fn get_meta_mut(&mut self) -> &mut PhysicalPlanMeta { + &mut self.meta + } + + fn output_schema(&self) -> Result { + Ok(DataSchemaRef::default()) + } + + fn children<'a>(&'a self) -> Box> + 'a> { + Box::new(std::iter::once(&self.input)) + } + + fn children_mut<'a>(&'a self) -> Box> + 'a> { + Box::new(std::iter::once(&mut self.input)) + } } diff --git a/src/query/sql/src/executor/physical_plans/physical_row_fetch.rs b/src/query/sql/src/executor/physical_plans/physical_row_fetch.rs index b82415a7b34bc..1c0bab38653f2 100644 --- a/src/query/sql/src/executor/physical_plans/physical_row_fetch.rs +++ b/src/query/sql/src/executor/physical_plans/physical_row_fetch.rs @@ -62,12 +62,9 @@ impl IPhysicalPlan for RowFetch { fn children_mut<'a>(&'a self) -> Box> + 'a> { Box::new(std::iter::once(&mut self.input)) } -} -impl RowFetch { - pub fn output_schema(&self) -> Result { - let mut fields = self.input.output_schema()?.fields().clone(); - fields.extend_from_slice(&self.fetched_fields); - Ok(DataSchemaRefExt::create(fields)) + #[recursive::recursive] + fn try_find_single_data_source(&self) -> Option<&DataSourcePlan> { + self.input.try_find_single_data_source() } } diff --git a/src/query/sql/src/executor/physical_plans/physical_sort.rs b/src/query/sql/src/executor/physical_plans/physical_sort.rs index 915c1c38b7046..1ba82321ed8fd 100644 --- a/src/query/sql/src/executor/physical_plans/physical_sort.rs +++ b/src/query/sql/src/executor/physical_plans/physical_sort.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use databend_common_catalog::plan::DataSourcePlan; use databend_common_exception::Result; use databend_common_expression::types::DataType; use databend_common_expression::DataField; @@ -104,6 +105,11 @@ impl IPhysicalPlan for Sort { fn children_mut<'a>(&'a self) -> Box> + 'a> { Box::new(std::iter::once(&mut self.input)) } + + #[recursive::recursive] + fn try_find_single_data_source(&self) -> Option<&DataSourcePlan> { + self.input.try_find_single_data_source() + } } impl Sort { @@ -119,44 +125,6 @@ impl Sort { } Ok(DataType::Binary) } - - pub fn output_schema(&self) -> Result { - let input_schema = self.input.output_schema()?; - let mut fields = input_schema.fields().clone(); - if matches!(self.after_exchange, Some(true)) { - // If the plan is after exchange plan in cluster mode, - // the order column is at the last of the input schema. - debug_assert_eq!(fields.last().unwrap().name(), ORDER_COL_NAME); - debug_assert_eq!( - fields.last().unwrap().data_type(), - &self.order_col_type(&input_schema)? - ); - fields.pop(); - } else { - if let Some(proj) = &self.pre_projection { - let fileted_fields = proj - .iter() - .filter_map(|index| input_schema.field_with_name(&index.to_string()).ok()) - .cloned() - .collect::>(); - if fileted_fields.len() < fields.len() { - // Only if the projection is not a full projection, we need to add a projection transform. - fields = fileted_fields - } - } - - if matches!(self.after_exchange, Some(false)) { - // If the plan is before exchange plan in cluster mode, - // the order column should be added to the output schema. - fields.push(DataField::new( - ORDER_COL_NAME, - self.order_col_type(&input_schema)?, - )); - } - } - - Ok(DataSchemaRefExt::create(fields)) - } } impl PhysicalPlanBuilder { diff --git a/src/query/sql/src/executor/physical_plans/physical_table_scan.rs b/src/query/sql/src/executor/physical_plans/physical_table_scan.rs index 16067b8b1e9b1..4b59931acc661 100644 --- a/src/query/sql/src/executor/physical_plans/physical_table_scan.rs +++ b/src/query/sql/src/executor/physical_plans/physical_table_scan.rs @@ -94,6 +94,32 @@ impl IPhysicalPlan for TableScan { &mut self.meta } + fn output_schema(&self) -> Result { + let schema = self.source.schema(); + let mut fields = Vec::with_capacity(self.name_mapping.len()); + let mut name_and_ids = self.name_mapping + .iter() + .map(|(name, id)| { + let index = schema.index_of(name)?; + Ok((name, id, index)) + }) + .collect::>>()?; + // Make the order of output fields the same as their indexes in te table schema. + name_and_ids.sort_by_key(|(_, _, index)| *index); + + for (name, id, _) in name_and_ids { + let orig_field = schema.field_with_name(name)?; + let data_type = DataType::from(orig_field.data_type()); + fields.push(DataField::new(&id.to_string(), data_type)); + } + + Ok(DataSchemaRefExt::create(fields)) + } + + fn try_find_single_data_source(&self) -> Option<&DataSourcePlan> { + Some(&self.source) + } + fn get_all_data_source(&self, sources: &mut Vec<(u32, Box)>) { sources.push((self.get_id(), self.source.clone())); } @@ -132,11 +158,6 @@ impl TableScan { } Ok(fields) } - - pub fn output_schema(&self) -> Result { - let fields = TableScan::output_fields(self.source.schema(), &self.name_mapping)?; - Ok(DataSchemaRefExt::create(fields)) - } } impl PhysicalPlanBuilder { diff --git a/src/query/sql/src/executor/physical_plans/physical_udf.rs b/src/query/sql/src/executor/physical_plans/physical_udf.rs index 02859a8091cb4..f8bf1534a05d2 100644 --- a/src/query/sql/src/executor/physical_plans/physical_udf.rs +++ b/src/query/sql/src/executor/physical_plans/physical_udf.rs @@ -13,7 +13,7 @@ // limitations under the License. use std::collections::BTreeMap; - +use databend_common_catalog::plan::DataSourcePlan; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_expression::types::DataType; @@ -23,7 +23,7 @@ use databend_common_expression::DataSchemaRefExt; use databend_common_functions::BUILTIN_FUNCTIONS; use crate::executor::explain::PlanStatsInfo; -use crate::executor::{IPhysicalPlan, PhysicalPlan}; +use crate::executor::{IPhysicalPlan, PhysicalPlan, PhysicalPlanMeta}; use crate::executor::PhysicalPlanBuilder; use crate::optimizer::ir::SExpr; use crate::plans::UDFType; @@ -35,7 +35,8 @@ use crate::ScalarExpr; pub struct Udf { // A unique id of operator in a `PhysicalPlan` tree, only used for display. pub plan_id: u32, - pub input: Box, + meta: PhysicalPlanMeta, + pub input: Box, pub udf_funcs: Vec, pub script_udf: bool, // Only used for explain @@ -43,11 +44,15 @@ pub struct Udf { } impl IPhysicalPlan for Udf { + fn get_meta(&self) -> &PhysicalPlanMeta { + &self.meta + } -} + fn get_meta_mut(&mut self) -> &mut PhysicalPlanMeta { + &mut self.meta + } -impl Udf { - pub fn output_schema(&self) -> Result { + fn output_schema(&self) -> Result { let input_schema = self.input.output_schema()?; let mut fields = input_schema.fields().clone(); for udf_func in self.udf_funcs.iter() { @@ -57,6 +62,19 @@ impl Udf { } Ok(DataSchemaRefExt::create(fields)) } + + fn children<'a>(&'a self) -> Box> + 'a> { + Box::new(std::iter::once(&self.input)) + } + + fn children_mut<'a>(&'a self) -> Box> + 'a> { + Box::new(std::iter::once(&mut self.input)) + } + + #[recursive::recursive] + fn try_find_single_data_source(&self) -> Option<&DataSourcePlan> { + self.input.try_find_single_data_source() + } } #[derive(Clone, Debug, Eq, Hash, PartialEq, serde::Serialize, serde::Deserialize)] diff --git a/src/query/sql/src/executor/physical_plans/physical_union_all.rs b/src/query/sql/src/executor/physical_plans/physical_union_all.rs index 4d365dff18cc2..12e8b2c558334 100644 --- a/src/query/sql/src/executor/physical_plans/physical_union_all.rs +++ b/src/query/sql/src/executor/physical_plans/physical_union_all.rs @@ -20,7 +20,7 @@ use databend_common_expression::DataSchemaRefExt; use databend_common_expression::RemoteExpr; use crate::executor::explain::PlanStatsInfo; -use crate::executor::{IPhysicalPlan, PhysicalPlan}; +use crate::executor::{IPhysicalPlan, PhysicalPlan, PhysicalPlanMeta}; use crate::executor::PhysicalPlanBuilder; use crate::optimizer::ir::SExpr; use crate::ColumnSet; @@ -32,6 +32,7 @@ use crate::TypeCheck; pub struct UnionAll { // A unique id of operator in a `PhysicalPlan` tree, only used for display. pub plan_id: u32, + meta: PhysicalPlanMeta, pub left: Box, pub right: Box, pub left_outputs: Vec<(IndexType, Option)>, @@ -44,13 +45,25 @@ pub struct UnionAll { } impl IPhysicalPlan for UnionAll { + fn get_meta(&self) -> &PhysicalPlanMeta { + &self.meta + } -} + fn get_meta_mut(&mut self) -> &mut PhysicalPlanMeta { + &mut self.meta + } -impl UnionAll { - pub fn output_schema(&self) -> Result { + fn output_schema(&self) -> Result { Ok(self.schema.clone()) } + + fn children<'a>(&'a self) -> Box> + 'a> { + Box::new(std::iter::once(&self.left).chain(std::iter::once(&self.right))) + } + + fn children_mut<'a>(&'a self) -> Box> + 'a> { + Box::new(std::iter::once(&mut self.left).chain(std::iter::once(&mut self.right))) + } } impl PhysicalPlanBuilder { diff --git a/src/query/sql/src/executor/physical_plans/physical_window.rs b/src/query/sql/src/executor/physical_plans/physical_window.rs index 2cba6f8c10fb2..99ef9dcf6c2f4 100644 --- a/src/query/sql/src/executor/physical_plans/physical_window.rs +++ b/src/query/sql/src/executor/physical_plans/physical_window.rs @@ -13,7 +13,7 @@ // limitations under the License. use std::fmt::Display; - +use databend_common_catalog::plan::DataSourcePlan; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_expression::type_check; @@ -87,18 +87,10 @@ impl IPhysicalPlan for Window { fn children_mut<'a>(&'a self) -> Box> + 'a> { Box::new(std::iter::once(&mut self.input)) } -} -impl Window { - pub fn output_schema(&self) -> Result { - let input_schema = self.input.output_schema()?; - let mut fields = Vec::with_capacity(input_schema.fields().len() + 1); - fields.extend_from_slice(input_schema.fields()); - fields.push(DataField::new( - &self.index.to_string(), - self.func.data_type()?, - )); - Ok(DataSchemaRefExt::create(fields)) + #[recursive::recursive] + fn try_find_single_data_source(&self) -> Option<&DataSourcePlan> { + self.input.try_find_single_data_source() } } diff --git a/src/query/sql/src/executor/physical_plans/physical_window_partition.rs b/src/query/sql/src/executor/physical_plans/physical_window_partition.rs index 610e43d0c4064..d26a5f4084a99 100644 --- a/src/query/sql/src/executor/physical_plans/physical_window_partition.rs +++ b/src/query/sql/src/executor/physical_plans/physical_window_partition.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use databend_common_catalog::plan::DataSourcePlan; use databend_common_exception::Result; use databend_common_expression::DataSchemaRef; @@ -49,6 +50,11 @@ impl IPhysicalPlan for WindowPartition { fn children_mut<'a>(&'a self) -> Box> + 'a> { Box::new(std::iter::once(&mut self.input)) } + + #[recursive::recursive] + fn try_find_single_data_source(&self) -> Option<&DataSourcePlan> { + self.input.try_find_single_data_source() + } } #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] @@ -63,9 +69,3 @@ pub enum WindowPartitionTopNFunc { Rank, DenseRank, } - -impl WindowPartition { - pub fn output_schema(&self) -> Result { - self.input.output_schema() - } -} From ed320f893518110d2e3645f47ed3f4b27457f80e Mon Sep 17 00:00:00 2001 From: zhang2014 Date: Mon, 30 Jun 2025 08:25:18 +0800 Subject: [PATCH 3/8] refactor(query): use trait to refactor physical plan --- Cargo.lock | 1 + .../src/schedulers/fragments/fragmenter.rs | 152 ++++---- src/query/sql/Cargo.toml | 1 + src/query/sql/src/executor/physical_plan.rs | 345 +++--------------- .../sql/src/executor/physical_plan_builder.rs | 10 +- .../sql/src/executor/physical_plan_visitor.rs | 165 +-------- .../physical_add_stream_column.rs | 24 +- .../physical_aggregate_expand.rs | 23 +- .../physical_aggregate_final.rs | 201 +++++----- .../physical_aggregate_partial.rs | 30 +- .../physical_plans/physical_asof_join.rs | 4 +- .../physical_plans/physical_async_func.rs | 22 +- .../physical_plans/physical_broadcast.rs | 35 +- .../physical_plans/physical_cache_scan.rs | 11 +- .../physical_column_mutation.rs | 6 +- .../physical_plans/physical_commit_sink.rs | 6 +- .../physical_plans/physical_compact_source.rs | 24 +- .../physical_constant_table_scan.rs | 13 +- .../physical_copy_into_location.rs | 6 +- .../physical_copy_into_table.rs | 7 +- .../physical_distributed_insert_select.rs | 8 +- .../physical_plans/physical_eval_scalar.rs | 43 ++- .../physical_plans/physical_exchange.rs | 14 +- .../physical_plans/physical_exchange_sink.rs | 7 +- .../physical_exchange_source.rs | 5 +- .../physical_expression_scan.rs | 13 +- .../physical_plans/physical_filter.rs | 45 ++- .../physical_plans/physical_hash_join.rs | 89 ++++- .../executor/physical_plans/physical_join.rs | 4 +- .../executor/physical_plans/physical_limit.rs | 48 ++- .../physical_multi_table_insert.rs | 54 +-- .../physical_plans/physical_mutation.rs | 119 +++--- .../physical_mutation_into_organize.rs | 6 +- .../physical_mutation_into_split.rs | 6 +- .../physical_mutation_manipulate.rs | 6 +- .../physical_mutation_source.rs | 11 +- .../physical_plans/physical_project_set.rs | 20 +- .../physical_plans/physical_r_cte_scan.rs | 11 +- .../physical_plans/physical_range_join.rs | 43 ++- .../physical_plans/physical_recluster.rs | 10 +- .../physical_replace_async_source.rs | 4 +- .../physical_replace_deduplicate.rs | 6 +- .../physical_plans/physical_replace_into.rs | 8 +- .../physical_plans/physical_row_fetch.rs | 14 +- .../executor/physical_plans/physical_sort.rs | 38 +- .../physical_plans/physical_table_scan.rs | 99 +++-- .../executor/physical_plans/physical_udf.rs | 22 +- .../physical_plans/physical_union_all.rs | 30 +- .../physical_plans/physical_window.rs | 38 +- .../physical_window_partition.rs | 6 +- 50 files changed, 922 insertions(+), 991 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e892d62b79e4e..092cc3df02d6f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4197,6 +4197,7 @@ dependencies = [ "similar", "simsearch", "tokio", + "typetag", "unicase", "url", ] diff --git a/src/query/service/src/schedulers/fragments/fragmenter.rs b/src/query/service/src/schedulers/fragments/fragmenter.rs index 2e7a6e878b819..090f612c558d2 100644 --- a/src/query/service/src/schedulers/fragments/fragmenter.rs +++ b/src/query/service/src/schedulers/fragments/fragmenter.rs @@ -155,6 +155,11 @@ impl Fragmenter { } impl PhysicalPlanReplacer for Fragmenter { + fn replace_recluster(&mut self, plan: &Recluster) -> Result { + self.state = State::Recluster; + Ok(PhysicalPlan::Recluster(Box::new(plan.clone()))) + } + fn replace_table_scan(&mut self, plan: &TableScan) -> Result { self.state = State::SelectLeaf; Ok(PhysicalPlan::TableScan(plan.clone())) @@ -165,55 +170,6 @@ impl PhysicalPlanReplacer for Fragmenter { Ok(PhysicalPlan::ConstantTableScan(plan.clone())) } - fn replace_mutation_source(&mut self, plan: &MutationSource) -> Result { - self.state = State::MutationSource; - Ok(PhysicalPlan::MutationSource(plan.clone())) - } - - fn replace_mutation(&mut self, plan: &Mutation) -> Result { - let input = self.replace(&plan.input)?; - Ok(PhysicalPlan::Mutation(Box::new(Mutation { - input: Box::new(input), - ..plan.clone() - }))) - } - - fn replace_replace_into(&mut self, plan: &ReplaceInto) -> Result { - let input = self.replace(&plan.input)?; - self.state = State::ReplaceInto; - Ok(PhysicalPlan::ReplaceInto(Box::new(ReplaceInto { - input: Box::new(input), - ..plan.clone() - }))) - } - - // TODO(Sky): remove redundant code - fn replace_copy_into_table(&mut self, plan: &CopyIntoTable) -> Result { - match &plan.source { - CopyIntoTableSource::Stage(_) => { - self.state = State::SelectLeaf; - Ok(PhysicalPlan::CopyIntoTable(Box::new(plan.clone()))) - } - CopyIntoTableSource::Query(query_physical_plan) => { - let input = self.replace(query_physical_plan)?; - Ok(PhysicalPlan::CopyIntoTable(Box::new(CopyIntoTable { - source: CopyIntoTableSource::Query(Box::new(input)), - ..plan.clone() - }))) - } - } - } - - fn replace_recluster(&mut self, plan: &Recluster) -> Result { - self.state = State::Recluster; - Ok(PhysicalPlan::Recluster(Box::new(plan.clone()))) - } - - fn replace_compact_source(&mut self, plan: &CompactSource) -> Result { - self.state = State::Compact; - Ok(PhysicalPlan::CompactSource(Box::new(plan.clone()))) - } - fn replace_hash_join(&mut self, plan: &HashJoin) -> Result { let mut fragments = vec![]; let build_input = self.replace(plan.build.as_ref())?; @@ -250,33 +206,6 @@ impl PhysicalPlanReplacer for Fragmenter { })) } - fn replace_union(&mut self, plan: &UnionAll) -> Result { - let mut fragments = vec![]; - let left_input = self.replace(plan.left.as_ref())?; - let left_state = self.state.clone(); - - // Consume current fragments to prevent them being consumed by `right_input`. - fragments.append(&mut self.fragments); - let right_input = self.replace(plan.right.as_ref())?; - let right_state = self.state.clone(); - - fragments.append(&mut self.fragments); - self.fragments = fragments; - - // If any of the input is a source fragment, the union all is a source fragment. - if left_state == State::SelectLeaf || right_state == State::SelectLeaf { - self.state = State::SelectLeaf; - } else { - self.state = State::Other; - } - - Ok(PhysicalPlan::UnionAll(UnionAll { - left: Box::new(left_input), - right: Box::new(right_input), - ..plan.clone() - })) - } - fn replace_exchange(&mut self, plan: &Exchange) -> Result { // Recursively rewrite input let input = self.replace(plan.input.as_ref())?; @@ -341,4 +270,75 @@ impl PhysicalPlanReplacer for Fragmenter { source_fragment_id, })) } + + fn replace_union(&mut self, plan: &UnionAll) -> Result { + let mut fragments = vec![]; + let left_input = self.replace(plan.left.as_ref())?; + let left_state = self.state.clone(); + + // Consume current fragments to prevent them being consumed by `right_input`. + fragments.append(&mut self.fragments); + let right_input = self.replace(plan.right.as_ref())?; + let right_state = self.state.clone(); + + fragments.append(&mut self.fragments); + self.fragments = fragments; + + // If any of the input is a source fragment, the union all is a source fragment. + if left_state == State::SelectLeaf || right_state == State::SelectLeaf { + self.state = State::SelectLeaf; + } else { + self.state = State::Other; + } + + Ok(PhysicalPlan::UnionAll(UnionAll { + left: Box::new(left_input), + right: Box::new(right_input), + ..plan.clone() + })) + } + + // TODO(Sky): remove redundant code + fn replace_copy_into_table(&mut self, plan: &CopyIntoTable) -> Result { + match &plan.source { + CopyIntoTableSource::Stage(_) => { + self.state = State::SelectLeaf; + Ok(PhysicalPlan::CopyIntoTable(Box::new(plan.clone()))) + } + CopyIntoTableSource::Query(query_physical_plan) => { + let input = self.replace(query_physical_plan)?; + Ok(PhysicalPlan::CopyIntoTable(Box::new(CopyIntoTable { + source: CopyIntoTableSource::Query(Box::new(input)), + ..plan.clone() + }))) + } + } + } + + fn replace_compact_source(&mut self, plan: &CompactSource) -> Result { + self.state = State::Compact; + Ok(PhysicalPlan::CompactSource(Box::new(plan.clone()))) + } + + fn replace_replace_into(&mut self, plan: &ReplaceInto) -> Result { + let input = self.replace(&plan.input)?; + self.state = State::ReplaceInto; + Ok(PhysicalPlan::ReplaceInto(Box::new(ReplaceInto { + input: Box::new(input), + ..plan.clone() + }))) + } + + fn replace_mutation_source(&mut self, plan: &MutationSource) -> Result { + self.state = State::MutationSource; + Ok(PhysicalPlan::MutationSource(plan.clone())) + } + + fn replace_mutation(&mut self, plan: &Mutation) -> Result { + let input = self.replace(&plan.input)?; + Ok(PhysicalPlan::Mutation(Box::new(Mutation { + input: Box::new(input), + ..plan.clone() + }))) + } } diff --git a/src/query/sql/Cargo.toml b/src/query/sql/Cargo.toml index 1ced7b5eb052e..2fd6c166f24f4 100644 --- a/src/query/sql/Cargo.toml +++ b/src/query/sql/Cargo.toml @@ -47,6 +47,7 @@ chrono-tz = { workspace = true } cidr = { workspace = true } cron = { workspace = true } ctor = { workspace = true } +typetag = { workspace = true } dashmap = { workspace = true } derive-visitor = { workspace = true } educe = { workspace = true } diff --git a/src/query/sql/src/executor/physical_plan.rs b/src/query/sql/src/executor/physical_plan.rs index cec50f8837928..a6d8fc2fe3047 100644 --- a/src/query/sql/src/executor/physical_plan.rs +++ b/src/query/sql/src/executor/physical_plan.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::any::Any; use std::collections::HashMap; use std::fmt::Debug; use databend_common_catalog::plan::DataSourceInfo; @@ -83,33 +84,16 @@ use crate::executor::physical_plans::WindowPartition; pub struct PhysicalPlanMeta { plan_id: u32, name: String, - // for profile - desc: String, - // for profile - labels: HashMap>, } impl PhysicalPlanMeta { - pub fn new(name: String) -> PhysicalPlanMeta { - PhysicalPlanMeta { - plan_id: 0, - name, - desc: String::new(), - labels: HashMap::new(), - } - } - - pub fn with_desc(name: String, desc: String, labels: HashMap>) -> PhysicalPlanMeta { - PhysicalPlanMeta { - plan_id: 0, - name, - desc, - labels, - } + pub fn new(name: impl Into) -> PhysicalPlanMeta { + PhysicalPlanMeta { plan_id: 0, name } } } -pub trait IPhysicalPlan: serde::Serialize + serde::Deserialize + Clone + Debug { +#[typetag::serde] +pub trait IPhysicalPlan: Debug { fn get_meta(&self) -> &PhysicalPlanMeta; fn get_meta_mut(&mut self) -> &mut PhysicalPlanMeta; @@ -127,7 +111,7 @@ pub trait IPhysicalPlan: serde::Serialize + serde::Deserialize + Clone + Debug { /// Adjust the plan_id of the physical plan. /// This function will assign a unique plan_id to each physical plan node in a top-down manner. /// Which means the plan_id of a node is always greater than the plan_id of its parent node. - #[recursive::recursive] + // #[recursive::recursive] fn adjust_plan_id(&mut self, next_id: &mut u32) { self.get_meta_mut().plan_id = *next_id; *next_id += 1; @@ -144,11 +128,11 @@ pub trait IPhysicalPlan: serde::Serialize + serde::Deserialize + Clone + Debug { } } - fn children<'a>(&'a self) -> Box> + 'a> { + fn children(&self) -> Box> + '_> { Box::new(std::iter::empty()) } - fn children_mut<'a>(&'a self) -> Box> + 'a> { + fn children_mut(&mut self) -> Box> + '_> { Box::new(std::iter::empty()) } @@ -157,7 +141,7 @@ pub trait IPhysicalPlan: serde::Serialize + serde::Deserialize + Clone + Debug { None } - #[recursive::recursive] + // #[recursive::recursive] fn try_find_mutation_source(&self) -> Option { for child in self.children() { if let Some(plan) = child.try_find_mutation_source() { @@ -168,30 +152,65 @@ pub trait IPhysicalPlan: serde::Serialize + serde::Deserialize + Clone + Debug { None } - #[recursive::recursive] + // #[recursive::recursive] fn get_all_data_source(&self, sources: &mut Vec<(u32, Box)>) { for child in self.children() { child.get_all_data_source(sources); } } - #[recursive::recursive] + // #[recursive::recursive] fn set_pruning_stats(&mut self, stats: &mut HashMap) { for child in self.children_mut() { child.set_pruning_stats(stats) } } - #[recursive::recursive] + // #[recursive::recursive] fn is_distributed_plan(&self) -> bool { self.children().any(|child| child.is_distributed_plan()) } - #[recursive::recursive] + // #[recursive::recursive] fn is_warehouse_distributed_plan(&self) -> bool { self.children() .any(|child| child.is_warehouse_distributed_plan()) } + + fn get_desc(&self) -> Result { + Ok(String::new()) + } + + fn get_labels(&self) -> Result>> { + Ok(HashMap::new()) + } +} + +pub trait PhysicalPlanExt { + fn clone_box(&self) -> Box; + + fn visit(&self, visitor: Box) -> Result<()>; +} + +impl PhysicalPlanExt for T +where + T: 'static + IPhysicalPlan + Clone, +{ + fn clone_box(&self) -> Box { + Box::new(self.clone()) + } + + fn visit(&self, visitor: Box) -> Result<()> { + visitor.downcast() + // visitor.downcast() + todo!() + } +} + +impl Clone for Box { + fn clone(&self) -> Self { + self.clone_box() + } } #[derive(serde::Serialize, serde::Deserialize, Educe, EnumAsInner)] @@ -341,272 +360,4 @@ impl PhysicalPlan { PhysicalPlan::BroadcastSink(_) => "RuntimeFilterSink".to_string(), } } - - pub fn get_desc(&self) -> Result { - Ok(match self { - PhysicalPlan::TableScan(v) => format!( - "{}.{}", - v.source.source_info.catalog_name(), - v.source.source_info.desc() - ), - PhysicalPlan::Filter(v) => match v.predicates.is_empty() { - true => String::new(), - false => v.predicates[0].as_expr(&BUILTIN_FUNCTIONS).sql_display(), - }, - PhysicalPlan::AggregatePartial(v) => { - v.agg_funcs.iter().map(|x| x.display.clone()).join(", ") - } - PhysicalPlan::AggregateFinal(v) => { - v.agg_funcs.iter().map(|x| x.display.clone()).join(", ") - } - PhysicalPlan::Sort(v) => v - .order_by - .iter() - .map(|x| { - format!( - "{}{}{}", - x.display_name, - if x.asc { "" } else { " DESC" }, - if x.nulls_first { " NULLS FIRST" } else { "" }, - ) - }) - .join(", "), - PhysicalPlan::Limit(v) => match v.limit { - Some(limit) => format!("LIMIT {} OFFSET {}", limit, v.offset), - None => format!("OFFSET {}", v.offset), - }, - PhysicalPlan::EvalScalar(v) => v - .exprs - .iter() - .map(|(x, _)| x.as_expr(&BUILTIN_FUNCTIONS).sql_display()) - .join(", "), - PhysicalPlan::HashJoin(v) => { - let mut conditions = v - .build_keys - .iter() - .zip(v.probe_keys.iter()) - .map(|(l, r)| { - format!( - "({} = {})", - l.as_expr(&BUILTIN_FUNCTIONS).sql_display(), - r.as_expr(&BUILTIN_FUNCTIONS).sql_display() - ) - }) - .collect::>(); - - conditions.extend( - v.non_equi_conditions - .iter() - .map(|x| x.as_expr(&BUILTIN_FUNCTIONS).sql_display()), - ); - - conditions.join(" AND ") - } - PhysicalPlan::ProjectSet(v) => v - .srf_exprs - .iter() - .map(|(x, _)| x.as_expr(&BUILTIN_FUNCTIONS).sql_display()) - .join(", "), - PhysicalPlan::AggregateExpand(v) => v - .grouping_sets - .sets - .iter() - .map(|set| { - set.iter() - .map(|x| x.to_string()) - .collect::>() - .join(", ") - }) - .map(|s| format!("({})", s)) - .collect::>() - .join(", "), - PhysicalPlan::Window(v) => { - let partition_by = v - .partition_by - .iter() - .map(|x| x.to_string()) - .collect::>() - .join(", "); - - let order_by = v - .order_by - .iter() - .map(|x| { - format!( - "{}{}{}", - x.display_name, - if x.asc { "" } else { " DESC" }, - if x.nulls_first { " NULLS FIRST" } else { "" }, - ) - }) - .collect::>() - .join(", "); - - format!("partition by {}, order by {}", partition_by, order_by) - } - PhysicalPlan::RowFetch(v) => { - let table_schema = v.source.source_info.schema(); - let projected_schema = v.cols_to_fetch.project_schema(&table_schema); - projected_schema.fields.iter().map(|f| f.name()).join(", ") - } - PhysicalPlan::RangeJoin(v) => { - let mut condition = v - .conditions - .iter() - .map(|condition| { - let left = condition - .left_expr - .as_expr(&BUILTIN_FUNCTIONS) - .sql_display(); - let right = condition - .right_expr - .as_expr(&BUILTIN_FUNCTIONS) - .sql_display(); - format!("{left} {:?} {right}", condition.operator) - }) - .collect::>(); - - condition.extend( - v.other_conditions - .iter() - .map(|x| x.as_expr(&BUILTIN_FUNCTIONS).sql_display()), - ); - - condition.join(" AND ") - } - PhysicalPlan::Udf(v) => v - .udf_funcs - .iter() - .map(|x| format!("{}({})", x.func_name, x.arg_exprs.join(", "))) - .join(", "), - PhysicalPlan::UnionAll(v) => v - .left_outputs - .iter() - .zip(v.right_outputs.iter()) - .map(|(l, r)| format!("#{} <- #{}", l.0, r.0)) - .join(", "), - PhysicalPlan::AsyncFunction(v) => v - .async_func_descs - .iter() - .map(|x| x.display_name.clone()) - .join(", "), - _ => String::new(), - }) - } - - pub fn get_labels(&self) -> Result>> { - let mut labels = HashMap::with_capacity(16); - - match self { - PhysicalPlan::TableScan(v) => { - labels.insert(String::from("Full table name"), vec![format!( - "{}.{}", - v.source.source_info.catalog_name(), - v.source.source_info.desc() - )]); - - labels.insert( - format!( - "Columns ({} / {})", - v.output_schema()?.num_fields(), - std::cmp::max( - v.output_schema()?.num_fields(), - v.source.source_info.schema().num_fields(), - ) - ), - v.name_mapping.keys().cloned().collect(), - ); - labels.insert(String::from("Total partitions"), vec![v - .source - .statistics - .partitions_total - .to_string()]); - } - PhysicalPlan::Filter(v) => { - labels.insert( - String::from("Filter condition"), - v.predicates - .iter() - .map(|x| x.as_expr(&BUILTIN_FUNCTIONS).sql_display()) - .collect(), - ); - } - PhysicalPlan::Limit(v) => { - labels.insert(String::from("Offset"), vec![v.offset.to_string()]); - - if let Some(limit) = v.limit { - labels.insert(String::from("Number of rows"), vec![limit.to_string()]); - } - } - PhysicalPlan::EvalScalar(v) => { - labels.insert( - String::from("List of Expressions"), - v.exprs - .iter() - .map(|(x, _)| x.as_expr(&BUILTIN_FUNCTIONS).sql_display()) - .collect(), - ); - } - PhysicalPlan::AggregatePartial(v) => { - if !v.group_by_display.is_empty() { - labels.insert(String::from("Grouping keys"), v.group_by_display.clone()); - } - - if !v.agg_funcs.is_empty() { - labels.insert( - String::from("Aggregate Functions"), - v.agg_funcs.iter().map(|x| x.display.clone()).collect(), - ); - } - } - PhysicalPlan::AggregateFinal(v) => { - if !v.group_by_display.is_empty() { - labels.insert(String::from("Grouping keys"), v.group_by_display.clone()); - } - - if !v.agg_funcs.is_empty() { - labels.insert( - String::from("Aggregate Functions"), - v.agg_funcs.iter().map(|x| x.display.clone()).collect(), - ); - } - } - PhysicalPlan::HashJoin(v) => { - labels.insert(String::from("Join Type"), vec![v.join_type.to_string()]); - - if !v.build_keys.is_empty() { - labels.insert( - String::from("Join Build Side Keys"), - v.build_keys - .iter() - .map(|x| x.as_expr(&BUILTIN_FUNCTIONS).sql_display()) - .collect(), - ); - } - - if !v.probe_keys.is_empty() { - labels.insert( - String::from("Join Probe Side Keys"), - v.probe_keys - .iter() - .map(|x| x.as_expr(&BUILTIN_FUNCTIONS).sql_display()) - .collect(), - ); - } - - if !v.non_equi_conditions.is_empty() { - labels.insert( - String::from("Join Conditions"), - v.non_equi_conditions - .iter() - .map(|x| x.as_expr(&BUILTIN_FUNCTIONS).sql_display()) - .collect(), - ); - } - } - _ => {} - }; - - Ok(labels) - } } diff --git a/src/query/sql/src/executor/physical_plan_builder.rs b/src/query/sql/src/executor/physical_plan_builder.rs index c50adc9f30db4..ee4850d20dda5 100644 --- a/src/query/sql/src/executor/physical_plan_builder.rs +++ b/src/query/sql/src/executor/physical_plan_builder.rs @@ -25,7 +25,7 @@ use databend_storages_common_table_meta::meta::TableMetaTimestamps; use databend_storages_common_table_meta::meta::TableSnapshot; use crate::executor::explain::PlanStatsInfo; -use crate::executor::PhysicalPlan; +use crate::executor::{IPhysicalPlan, PhysicalPlan}; use crate::optimizer::ir::RelExpr; use crate::optimizer::ir::SExpr; use crate::plans::RelOperator; @@ -62,7 +62,7 @@ impl PhysicalPlanBuilder { }) } - pub async fn build(&mut self, s_expr: &SExpr, required: ColumnSet) -> Result { + pub async fn build(&mut self, s_expr: &SExpr, required: ColumnSet) -> Result> { let mut plan = self.build_physical_plan(s_expr, required).await?; plan.adjust_plan_id(&mut 0); @@ -74,7 +74,7 @@ impl PhysicalPlanBuilder { &mut self, s_expr: &SExpr, required: ColumnSet, - ) -> Result { + ) -> Result> { // Build stat info. let stat_info = self.build_plan_stat_info(s_expr)?; match s_expr.plan() { @@ -96,9 +96,7 @@ impl PhysicalPlanBuilder { } RelOperator::Sort(sort) => self.build_sort(s_expr, sort, required, stat_info).await, RelOperator::Limit(limit) => self.build_limit(s_expr, limit, required, stat_info).await, - RelOperator::Exchange(exchange) => { - self.build_exchange(s_expr, exchange, required).await - } + RelOperator::Exchange(exchange) => self.build_exchange(s_expr, exchange, required).await, RelOperator::UnionAll(union_all) => { self.build_union_all(s_expr, union_all, required, stat_info) .await diff --git a/src/query/sql/src/executor/physical_plan_visitor.rs b/src/query/sql/src/executor/physical_plan_visitor.rs index 7267b7665e370..cdfc27a60e28f 100644 --- a/src/query/sql/src/executor/physical_plan_visitor.rs +++ b/src/query/sql/src/executor/physical_plan_visitor.rs @@ -13,7 +13,7 @@ // limitations under the License. use databend_common_exception::Result; - +use crate::executor::{IPhysicalPlan, PhysicalPlanMeta}; use super::physical_plans::AddStreamColumn; use super::physical_plans::BroadcastSink; use super::physical_plans::BroadcastSource; @@ -131,12 +131,12 @@ pub trait PhysicalPlanReplacer { Ok(PhysicalPlan::BroadcastSource(plan.clone())) } - fn replace_runtime_filter_sink(&mut self, plan: &BroadcastSink) -> Result { + fn replace_runtime_filter_sink(&mut self, plan: &BroadcastSink) -> Result> { let input = self.replace(&plan.input)?; - Ok(PhysicalPlan::BroadcastSink(BroadcastSink { - plan_id: plan.plan_id, + Ok(Box::new(BroadcastSink { broadcast_id: plan.broadcast_id, input: Box::new(input), + meta: PhysicalPlanMeta::new("BroadcastSink"), })) } @@ -175,12 +175,12 @@ pub trait PhysicalPlanReplacer { fn replace_filter(&mut self, plan: &Filter) -> Result { let input = self.replace(&plan.input)?; - Ok(PhysicalPlan::Filter(Filter { - plan_id: plan.plan_id, + Ok(Box::new(Filter { projections: plan.projections.clone(), input: Box::new(input), predicates: plan.predicates.clone(), stat_info: plan.stat_info.clone(), + meta: PhysicalPlanMeta::new("Filter"), })) } @@ -647,155 +647,18 @@ pub trait PhysicalPlanReplacer { impl PhysicalPlan { pub fn traverse<'a, 'b>( - plan: &'a PhysicalPlan, - pre_visit: &'b mut dyn FnMut(&'a PhysicalPlan) -> bool, - visit: &'b mut dyn FnMut(&'a PhysicalPlan), - post_visit: &'b mut dyn FnMut(&'a PhysicalPlan), + plan: &'a Box, + pre_visit: &'b mut dyn FnMut(&'a Box) -> bool, + visit: &'b mut dyn FnMut(&'a Box), + post_visit: &'b mut dyn FnMut(&'a Box), ) { if pre_visit(plan) { visit(plan); - match plan { - PhysicalPlan::TableScan(_) - | PhysicalPlan::ReplaceAsyncSourcer(_) - | PhysicalPlan::RecursiveCteScan(_) - | PhysicalPlan::ConstantTableScan(_) - | PhysicalPlan::ExpressionScan(_) - | PhysicalPlan::CacheScan(_) - | PhysicalPlan::Recluster(_) - | PhysicalPlan::HilbertPartition(_) - | PhysicalPlan::ExchangeSource(_) - | PhysicalPlan::CompactSource(_) - | PhysicalPlan::MutationSource(_) - | PhysicalPlan::BroadcastSource(_) => {} - PhysicalPlan::Filter(plan) => { - Self::traverse(&plan.input, pre_visit, visit, post_visit); - } - PhysicalPlan::EvalScalar(plan) => { - Self::traverse(&plan.input, pre_visit, visit, post_visit); - } - PhysicalPlan::AggregateExpand(plan) => { - Self::traverse(&plan.input, pre_visit, visit, post_visit); - } - PhysicalPlan::AggregatePartial(plan) => { - Self::traverse(&plan.input, pre_visit, visit, post_visit); - } - PhysicalPlan::AggregateFinal(plan) => { - Self::traverse(&plan.input, pre_visit, visit, post_visit); - } - PhysicalPlan::Window(plan) => { - Self::traverse(&plan.input, pre_visit, visit, post_visit); - } - PhysicalPlan::WindowPartition(plan) => { - Self::traverse(&plan.input, pre_visit, visit, post_visit); - } - PhysicalPlan::Sort(plan) => { - Self::traverse(&plan.input, pre_visit, visit, post_visit); - } - PhysicalPlan::Limit(plan) => { - Self::traverse(&plan.input, pre_visit, visit, post_visit); - } - PhysicalPlan::RowFetch(plan) => { - Self::traverse(&plan.input, pre_visit, visit, post_visit); - } - PhysicalPlan::HashJoin(plan) => { - Self::traverse(&plan.build, pre_visit, visit, post_visit); - Self::traverse(&plan.probe, pre_visit, visit, post_visit); - } - PhysicalPlan::Exchange(plan) => { - Self::traverse(&plan.input, pre_visit, visit, post_visit); - } - PhysicalPlan::ExchangeSink(plan) => { - Self::traverse(&plan.input, pre_visit, visit, post_visit); - } - PhysicalPlan::UnionAll(plan) => { - Self::traverse(&plan.left, pre_visit, visit, post_visit); - Self::traverse(&plan.right, pre_visit, visit, post_visit); - } - PhysicalPlan::DistributedInsertSelect(plan) => { - Self::traverse(&plan.input, pre_visit, visit, post_visit); - } - PhysicalPlan::ProjectSet(plan) => { - Self::traverse(&plan.input, pre_visit, visit, post_visit) - } - PhysicalPlan::CopyIntoTable(plan) => match &plan.source { - CopyIntoTableSource::Query(input) => { - Self::traverse(input, pre_visit, visit, post_visit); - } - CopyIntoTableSource::Stage(input) => { - Self::traverse(input, pre_visit, visit, post_visit); - } - }, - PhysicalPlan::CopyIntoLocation(plan) => { - Self::traverse(&plan.input, pre_visit, visit, post_visit) - } - PhysicalPlan::RangeJoin(plan) => { - Self::traverse(&plan.left, pre_visit, visit, post_visit); - Self::traverse(&plan.right, pre_visit, visit, post_visit); - } - PhysicalPlan::CommitSink(plan) => { - Self::traverse(&plan.input, pre_visit, visit, post_visit); - } - PhysicalPlan::ReplaceDeduplicate(plan) => { - Self::traverse(&plan.input, pre_visit, visit, post_visit); - } - PhysicalPlan::ReplaceInto(plan) => { - Self::traverse(&plan.input, pre_visit, visit, post_visit); - } - PhysicalPlan::ColumnMutation(plan) => { - Self::traverse(&plan.input, pre_visit, visit, post_visit); - } - PhysicalPlan::Mutation(plan) => { - Self::traverse(&plan.input, pre_visit, visit, post_visit); - } - PhysicalPlan::MutationSplit(plan) => { - Self::traverse(&plan.input, pre_visit, visit, post_visit); - } - PhysicalPlan::MutationManipulate(plan) => { - Self::traverse(&plan.input, pre_visit, visit, post_visit); - } - PhysicalPlan::MutationOrganize(plan) => { - Self::traverse(&plan.input, pre_visit, visit, post_visit); - } - PhysicalPlan::AddStreamColumn(plan) => { - Self::traverse(&plan.input, pre_visit, visit, post_visit); - } - PhysicalPlan::Udf(plan) => { - Self::traverse(&plan.input, pre_visit, visit, post_visit); - } - PhysicalPlan::AsyncFunction(plan) => { - Self::traverse(&plan.input, pre_visit, visit, post_visit); - } - PhysicalPlan::Duplicate(plan) => { - Self::traverse(&plan.input, pre_visit, visit, post_visit); - } - PhysicalPlan::Shuffle(plan) => { - Self::traverse(&plan.input, pre_visit, visit, post_visit); - } - PhysicalPlan::ChunkFilter(plan) => { - Self::traverse(&plan.input, pre_visit, visit, post_visit); - } - PhysicalPlan::ChunkEvalScalar(plan) => { - Self::traverse(&plan.input, pre_visit, visit, post_visit); - } - PhysicalPlan::ChunkCastSchema(plan) => { - Self::traverse(&plan.input, pre_visit, visit, post_visit); - } - PhysicalPlan::ChunkFillAndReorder(plan) => { - Self::traverse(&plan.input, pre_visit, visit, post_visit); - } - PhysicalPlan::ChunkAppendData(plan) => { - Self::traverse(&plan.input, pre_visit, visit, post_visit); - } - PhysicalPlan::ChunkMerge(plan) => { - Self::traverse(&plan.input, pre_visit, visit, post_visit); - } - PhysicalPlan::ChunkCommitInsert(plan) => { - Self::traverse(&plan.input, pre_visit, visit, post_visit); - } - PhysicalPlan::BroadcastSink(plan) => { - Self::traverse(&plan.input, pre_visit, visit, post_visit); - } + + for child in plan.children() { + Self::traverse(&child, pre_visit, visit, post_visit); } + post_visit(plan); } } diff --git a/src/query/sql/src/executor/physical_plans/physical_add_stream_column.rs b/src/query/sql/src/executor/physical_plans/physical_add_stream_column.rs index 9c774af387537..77d9ab7cb1630 100644 --- a/src/query/sql/src/executor/physical_plans/physical_add_stream_column.rs +++ b/src/query/sql/src/executor/physical_plans/physical_add_stream_column.rs @@ -37,14 +37,14 @@ use crate::Visibility; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct AddStreamColumn { - pub plan_id: u32, - meta: PhysicalPlanMeta, + pub meta: PhysicalPlanMeta, pub input: Box, pub exprs: Vec, pub projections: Vec, pub stream_columns: Vec, } +#[typetag::serde] impl IPhysicalPlan for AddStreamColumn { fn get_meta(&self) -> &PhysicalPlanMeta { &self.meta @@ -58,7 +58,7 @@ impl IPhysicalPlan for AddStreamColumn { Box::new(std::iter::once(&self.input)) } - fn children_mut<'a>(&'a self) -> Box> + 'a> { + fn children_mut<'a>(&'a mut self) -> Box> + 'a> { Box::new(std::iter::once(&mut self.input)) } } @@ -66,10 +66,10 @@ impl IPhysicalPlan for AddStreamColumn { impl AddStreamColumn { pub fn new( metadata: &MetadataRef, - input: PhysicalPlan, + input: Box, table_index: usize, table_version: u64, - ) -> Result { + ) -> Result> { let input_schema = input.output_schema()?; let num_fields = input_schema.fields().len(); let column_entries = metadata.read().columns_by_table_index(table_index); @@ -98,7 +98,7 @@ impl AddStreamColumn { Box::new(stream_column.data_type()), Visibility::Visible, ) - .build(), + .build(), }); let current_stream_column_scalar_expr = match stream_column.column_type() { @@ -119,7 +119,7 @@ impl AddStreamColumn { Box::new(stream_column.data_type()), Visibility::Visible, ) - .build(), + .build(), }) } StreamColumnType::OriginRowNum => { @@ -132,7 +132,7 @@ impl AddStreamColumn { Box::new(stream_column.data_type()), Visibility::Visible, ) - .build(), + .build(), }) } StreamColumnType::RowVersion => unreachable!(), @@ -175,12 +175,12 @@ impl AddStreamColumn { // ORIGIN_BLOCK_ROW_NUM, ORIGIN_BLOCK_ID. let stream_columns = vec![stream_columns[2].clone(), stream_columns[1].clone()]; - Ok(Self { - plan_id: 0, - input: Box::new(input), + Ok(Box::new(AddStreamColumn { + input, exprs, projections, stream_columns, - }) + meta: PhysicalPlanMeta::new("AddStreamColumn"), + })) } } diff --git a/src/query/sql/src/executor/physical_plans/physical_aggregate_expand.rs b/src/query/sql/src/executor/physical_plans/physical_aggregate_expand.rs index 5e2693f75c07a..fd676c62d3fdd 100644 --- a/src/query/sql/src/executor/physical_plans/physical_aggregate_expand.rs +++ b/src/query/sql/src/executor/physical_plans/physical_aggregate_expand.rs @@ -28,9 +28,7 @@ use crate::IndexType; /// Add dummy data before `GROUPING SETS`. #[derive(serde::Serialize, serde::Deserialize, Clone, Debug)] pub struct AggregateExpand { - // A unique id of operator in a `PhysicalPlan` tree, only used for display. - pub plan_id: u32, - meta: PhysicalPlanMeta, + pub meta: PhysicalPlanMeta, pub input: Box, pub group_bys: Vec, pub grouping_sets: GroupingSets, @@ -39,6 +37,7 @@ pub struct AggregateExpand { pub stat_info: Option, } +#[typetag::serde] impl IPhysicalPlan for AggregateExpand { fn get_meta(&self) -> &PhysicalPlanMeta { &self.meta @@ -79,7 +78,23 @@ impl IPhysicalPlan for AggregateExpand { Box::new(std::iter::once(&self.input)) } - fn children_mut<'a>(&'a self) -> Box> + 'a> { + fn children_mut<'a>(&'a mut self) -> Box> + 'a> { Box::new(std::iter::once(&mut self.input)) } + + fn get_desc(&self) -> Result { + Ok(self + .grouping_sets + .sets + .iter() + .map(|set| { + set.iter() + .map(|x| x.to_string()) + .collect::>() + .join(", ") + }) + .map(|s| format!("({})", s)) + .collect::>() + .join(", ")) + } } diff --git a/src/query/sql/src/executor/physical_plans/physical_aggregate_final.rs b/src/query/sql/src/executor/physical_plans/physical_aggregate_final.rs index 54bf09b8afe90..de30eb1bfc642 100644 --- a/src/query/sql/src/executor/physical_plans/physical_aggregate_final.rs +++ b/src/query/sql/src/executor/physical_plans/physical_aggregate_final.rs @@ -12,7 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::collections::HashMap; use std::sync::Arc; +use itertools::Itertools; use databend_common_catalog::plan::DataSourcePlan; use databend_common_exception::ErrorCode; use databend_common_exception::Result; @@ -40,8 +42,6 @@ use crate::ScalarExpr; #[derive(serde::Serialize, serde::Deserialize, Clone, Debug)] pub struct AggregateFinal { meta: PhysicalPlanMeta, - // A unique id of operator in a `PhysicalPlan` tree, only used for display. - pub plan_id: u32, pub input: Box, pub group_by: Vec, pub agg_funcs: Vec, @@ -52,6 +52,7 @@ pub struct AggregateFinal { pub stat_info: Option, } +#[typetag::serde] impl IPhysicalPlan for AggregateFinal { fn get_meta(&self) -> &PhysicalPlanMeta { &self.meta @@ -82,9 +83,29 @@ impl IPhysicalPlan for AggregateFinal { Box::new(std::iter::once(&self.input)) } - fn children_mut<'a>(&'a self) -> Box> + 'a> { + fn children_mut<'a>(&'a mut self) -> Box> + 'a> { Box::new(std::iter::once(&mut self.input)) } + + fn get_desc(&self) -> Result { + Ok(self.agg_funcs.iter().map(|x| x.display.clone()).join(", ")) + } + + fn get_labels(&self) -> Result>> { + let mut labels = HashMap::with_capacity(2); + if !self.group_by_display.is_empty() { + labels.insert(String::from("Grouping keys"), self.group_by_display.clone()); + } + + if !self.agg_funcs.is_empty() { + labels.insert( + String::from("Aggregate Functions"), + self.agg_funcs.iter().map(|x| x.display.clone()).collect(), + ); + } + + Ok(labels) + } } impl PhysicalPlanBuilder { @@ -94,7 +115,7 @@ impl PhysicalPlanBuilder { agg: &crate::plans::Aggregate, mut required: ColumnSet, stat_info: PlanStatsInfo, - ) -> Result { + ) -> Result> { // 1. Prune unused Columns. let mut used = vec![]; for item in &agg.aggregate_functions { @@ -250,12 +271,12 @@ impl PhysicalPlanBuilder { if let Some(grouping_sets) = agg.grouping_sets.as_ref() { assert_eq!(grouping_sets.dup_group_items.len(), group_items.len() - 1); // ignore `_grouping_id`. - // If the aggregation function argument if a group item, - // we cannot use the group item directly. - // It's because the group item will be wrapped with nullable and fill dummy NULLs (in `AggregateExpand` plan), - // which will cause panic while executing aggregation function. - // To avoid the panic, we will duplicate (`Arc::clone`) original group item columns in `AggregateExpand`, - // we should use these columns instead. + // If the aggregation function argument if a group item, + // we cannot use the group item directly. + // It's because the group item will be wrapped with nullable and fill dummy NULLs (in `AggregateExpand` plan), + // which will cause panic while executing aggregation function. + // To avoid the panic, we will duplicate (`Arc::clone`) original group item columns in `AggregateExpand`, + // we should use these columns instead. for func in agg_funcs.iter_mut() { for arg in func.arg_indices.iter_mut() { if let Some(pos) = group_items.iter().position(|g| g == arg) { @@ -280,84 +301,85 @@ impl PhysicalPlanBuilder { match input { PhysicalPlan::Exchange(Exchange { input, kind, .. }) - if group_by_shuffle_mode == "before_merge" => - { - let aggregate_partial = if let Some(grouping_sets) = agg.grouping_sets { - let expand = AggregateExpand { - plan_id: 0, - input, - group_bys: group_items.clone(), - grouping_sets, - stat_info: Some(stat_info.clone()), + if group_by_shuffle_mode == "before_merge" => + { + let aggregate_partial = if let Some(grouping_sets) = agg.grouping_sets { + let expand = AggregateExpand { + input, + grouping_sets, + group_bys: group_items.clone(), + stat_info: Some(stat_info.clone()), + meta: PhysicalPlanMeta::new("AggregateExpand"), + }; + + AggregatePartial { + input: Box::new(PhysicalPlan::AggregateExpand(expand)), + agg_funcs, + enable_experimental_aggregate_hashtable, + group_by_display, + group_by: group_items, + stat_info: Some(stat_info), + rank_limit: None, + meta: PhysicalPlanMeta::new("AggregatePartial"), + } + } else { + AggregatePartial { + input, + agg_funcs, + rank_limit, + group_by_display, + enable_experimental_aggregate_hashtable, + group_by: group_items, + stat_info: Some(stat_info), + meta: PhysicalPlanMeta::new("AggregatePartial"), + } }; - AggregatePartial { - plan_id: 0, - input: Box::new(PhysicalPlan::AggregateExpand(expand)), - agg_funcs, - enable_experimental_aggregate_hashtable, - group_by_display, - group_by: group_items, - stat_info: Some(stat_info), - rank_limit: None, - } - } else { - AggregatePartial { - plan_id: 0, - input, - agg_funcs, - enable_experimental_aggregate_hashtable, - group_by_display, - group_by: group_items, - stat_info: Some(stat_info), - rank_limit, - } - }; - - let keys = { - let schema = aggregate_partial.output_schema()?; - let end = schema.num_fields(); - let start = end - aggregate_partial.group_by.len(); - (start..end) - .map(|id| RemoteExpr::ColumnRef { - span: None, - id, - data_type: schema.field(id).data_type().clone(), - display_name: (id - start).to_string(), - }) - .collect() - }; - - PhysicalPlan::Exchange(Exchange { - plan_id: 0, - kind, - allow_adjust_parallelism: true, - ignore_exchange: false, - input: Box::new(PhysicalPlan::AggregatePartial(aggregate_partial)), - keys, - }) - } + + let keys = { + let schema = aggregate_partial.output_schema()?; + let end = schema.num_fields(); + let start = end - aggregate_partial.group_by.len(); + (start..end) + .map(|id| RemoteExpr::ColumnRef { + span: None, + id, + data_type: schema.field(id).data_type().clone(), + display_name: (id - start).to_string(), + }) + .collect() + }; + + PhysicalPlan::Exchange(Exchange { + kind, + keys, + ignore_exchange: false, + allow_adjust_parallelism: true, + meta: PhysicalPlanMeta::new("Exchange"), + input: Box::new(PhysicalPlan::AggregatePartial(aggregate_partial)), + }) + } _ => { if let Some(grouping_sets) = agg.grouping_sets { let expand = AggregateExpand { - plan_id: 0, + grouping_sets, input: Box::new(input), group_bys: group_items.clone(), - grouping_sets, stat_info: Some(stat_info.clone()), + meta: PhysicalPlanMeta::new("AggregateExpand"), }; - PhysicalPlan::AggregatePartial(AggregatePartial { - plan_id: 0, + + Box::new(AggregatePartial { agg_funcs, - enable_experimental_aggregate_hashtable, group_by_display, + enable_experimental_aggregate_hashtable, + rank_limit: None, group_by: group_items, - input: Box::new(PhysicalPlan::AggregateExpand(expand)), + input: Box::new(expand), stat_info: Some(stat_info), - rank_limit: None, + meta: PhysicalPlanMeta::new("AggregatePartial"), }) } else { PhysicalPlan::AggregatePartial(AggregatePartial { - plan_id: 0, agg_funcs, enable_experimental_aggregate_hashtable, group_by_display, @@ -365,6 +387,7 @@ impl PhysicalPlanBuilder { input: Box::new(input), stat_info: Some(stat_info), rank_limit, + meta: PhysicalPlanMeta::new("AggregatePartial"), }) } } @@ -377,9 +400,9 @@ impl PhysicalPlanBuilder { PhysicalPlan::AggregatePartial(ref agg) => agg.input.output_schema()?, PhysicalPlan::Exchange(Exchange { - input: box PhysicalPlan::AggregatePartial(ref agg), - .. - }) => agg.input.output_schema()?, + input: box PhysicalPlan::AggregatePartial(ref agg), + .. + }) => agg.input.output_schema()?, _ => { return Err(ErrorCode::Internal(format!( @@ -513,32 +536,30 @@ impl PhysicalPlanBuilder { let before_group_by_schema = partial.input.output_schema()?; PhysicalPlan::AggregateFinal(AggregateFinal { - plan_id: 0, - group_by_display: partial.group_by_display.clone(), - input: Box::new(input), - group_by: group_items, agg_funcs, before_group_by_schema, - + group_by: group_items, + input: Box::new(input), stat_info: Some(stat_info), + group_by_display: partial.group_by_display.clone(), + meta: PhysicalPlanMeta::new("AggregateFinal"), }) } PhysicalPlan::Exchange(Exchange { - input: box PhysicalPlan::AggregatePartial(ref partial), - .. - }) => { + input: box PhysicalPlan::AggregatePartial(ref partial), + .. + }) => { let before_group_by_schema = partial.input.output_schema()?; - PhysicalPlan::AggregateFinal(AggregateFinal { - plan_id: 0, - group_by_display: partial.group_by_display.clone(), - input: Box::new(input), - group_by: group_items, + Box::new(AggregateFinal { agg_funcs, before_group_by_schema, - + group_by: group_items, + input: Box::new(input), stat_info: Some(stat_info), + group_by_display: partial.group_by_display.clone(), + meta: PhysicalPlanMeta::new("AggregateFinal"), }) } diff --git a/src/query/sql/src/executor/physical_plans/physical_aggregate_partial.rs b/src/query/sql/src/executor/physical_plans/physical_aggregate_partial.rs index 50156679a8acc..a2261f8f146a8 100644 --- a/src/query/sql/src/executor/physical_plans/physical_aggregate_partial.rs +++ b/src/query/sql/src/executor/physical_plans/physical_aggregate_partial.rs @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::collections::HashMap; +use itertools::Itertools; use databend_common_catalog::plan::DataSourcePlan; use databend_common_exception::Result; use databend_common_expression::types::DataType; @@ -29,9 +31,7 @@ use crate::IndexType; #[derive(serde::Serialize, serde::Deserialize, Clone, Debug)] pub struct AggregatePartial { - meta: PhysicalPlanMeta, - // A unique id of operator in a `PhysicalPlan` tree, only used for display. - pub plan_id: u32, + pub meta: PhysicalPlanMeta, pub input: Box, pub group_by: Vec, pub agg_funcs: Vec, @@ -44,6 +44,7 @@ pub struct AggregatePartial { pub stat_info: Option, } +#[typetag::serde] impl IPhysicalPlan for AggregatePartial { fn get_meta(&self) -> &PhysicalPlanMeta { &self.meta @@ -78,7 +79,28 @@ impl IPhysicalPlan for AggregatePartial { Box::new(std::iter::once(&self.input)) } - fn children_mut<'a>(&'a self) -> Box> + 'a> { + fn children_mut<'a>(&'a mut self) -> Box> + 'a> { Box::new(std::iter::once(&mut self.input)) } + + fn get_desc(&self) -> Result { + Ok(self.agg_funcs.iter().map(|x| x.display.clone()).join(", ")) + } + + fn get_labels(&self) -> Result>> { + let mut labels = HashMap::with_capacity(2); + + if !self.group_by_display.is_empty() { + labels.insert(String::from("Grouping keys"), v.group_by_display.clone()); + } + + if !self.agg_funcs.is_empty() { + labels.insert( + String::from("Aggregate Functions"), + self.agg_funcs.iter().map(|x| x.display.clone()).collect(), + ); + } + + Ok(labels) + } } diff --git a/src/query/sql/src/executor/physical_plans/physical_asof_join.rs b/src/query/sql/src/executor/physical_plans/physical_asof_join.rs index 7cc90d45a2b11..eacc5747981e3 100644 --- a/src/query/sql/src/executor/physical_plans/physical_asof_join.rs +++ b/src/query/sql/src/executor/physical_plans/physical_asof_join.rs @@ -24,7 +24,7 @@ use crate::binder::bind_window_function_info; use crate::binder::ColumnBindingBuilder; use crate::binder::WindowFunctionInfo; use crate::binder::WindowOrderByInfo; -use crate::executor::PhysicalPlan; +use crate::executor::{IPhysicalPlan, PhysicalPlan}; use crate::executor::PhysicalPlanBuilder; use crate::optimizer::ir::RelExpr; use crate::optimizer::ir::SExpr; @@ -55,7 +55,7 @@ impl PhysicalPlanBuilder { required: (ColumnSet, ColumnSet), mut range_conditions: Vec, mut other_conditions: Vec, - ) -> Result { + ) -> Result> { let mut window_index: usize = 0; if range_conditions.is_empty() { diff --git a/src/query/sql/src/executor/physical_plans/physical_async_func.rs b/src/query/sql/src/executor/physical_plans/physical_async_func.rs index 5309bac843d58..b93cb5877ec5b 100644 --- a/src/query/sql/src/executor/physical_plans/physical_async_func.rs +++ b/src/query/sql/src/executor/physical_plans/physical_async_func.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use itertools::Itertools; use databend_common_catalog::plan::DataSourcePlan; use databend_common_exception::ErrorCode; use databend_common_exception::Result; @@ -31,15 +32,14 @@ use crate::ScalarExpr; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct AsyncFunction { - // A unique id of operator in a `PhysicalPlan` tree, only used for display. - pub plan_id: u32, - meta: PhysicalPlanMeta, + pub meta: PhysicalPlanMeta, pub input: Box, pub async_func_descs: Vec, // Only used for explain pub stat_info: Option, } +#[typetag::serde] impl IPhysicalPlan for AsyncFunction { fn get_meta(&self) -> &PhysicalPlanMeta { &self.meta @@ -64,7 +64,7 @@ impl IPhysicalPlan for AsyncFunction { Box::new(std::iter::once(&self.input)) } - fn children_mut<'a>(&'a self) -> Box> + 'a> { + fn children_mut<'a>(&'a mut self) -> Box> + 'a> { Box::new(std::iter::once(&mut self.input)) } @@ -72,6 +72,14 @@ impl IPhysicalPlan for AsyncFunction { fn try_find_single_data_source(&self) -> Option<&DataSourcePlan> { self.input.try_find_single_data_source() } + + fn get_desc(&self) -> Result { + Ok(self + .async_func_descs + .iter() + .map(|x| x.display_name.clone()) + .join(", ")) + } } #[derive(Clone, Debug, Eq, PartialEq, serde::Serialize, serde::Deserialize)] @@ -92,7 +100,7 @@ impl PhysicalPlanBuilder { async_func_plan: &crate::plans::AsyncFunction, mut required: ColumnSet, stat_info: PlanStatsInfo, - ) -> Result { + ) -> Result> { // 1. Prune unused Columns. let mut used = vec![]; for item in async_func_plan.items.iter() { @@ -151,8 +159,8 @@ impl PhysicalPlanBuilder { }) .collect::>>()?; - Ok(PhysicalPlan::AsyncFunction(AsyncFunction { - plan_id: 0, + Ok(Box::new(AsyncFunction { + meta: PhysicalPlanMeta::new("AsyncFunction"), input: Box::new(input), async_func_descs, stat_info: Some(stat_info), diff --git a/src/query/sql/src/executor/physical_plans/physical_broadcast.rs b/src/query/sql/src/executor/physical_plans/physical_broadcast.rs index 45a17a3f95a79..d38a22cfeade5 100644 --- a/src/query/sql/src/executor/physical_plans/physical_broadcast.rs +++ b/src/query/sql/src/executor/physical_plans/physical_broadcast.rs @@ -21,11 +21,11 @@ use crate::executor::{IPhysicalPlan, PhysicalPlan, PhysicalPlanMeta}; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct BroadcastSource { - pub plan_id: u32, - meta: PhysicalPlanMeta, + pub meta: PhysicalPlanMeta, pub broadcast_id: u32, } +#[typetag::serde] impl IPhysicalPlan for BroadcastSource { fn get_meta(&self) -> &PhysicalPlanMeta { &self.meta @@ -38,12 +38,12 @@ impl IPhysicalPlan for BroadcastSource { #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct BroadcastSink { - pub plan_id: u32, - meta: PhysicalPlanMeta, + pub meta: PhysicalPlanMeta, pub broadcast_id: u32, pub input: Box, } +#[typetag::serde] impl IPhysicalPlan for BroadcastSink { fn get_meta(&self) -> &PhysicalPlanMeta { &self.meta @@ -61,33 +61,34 @@ impl IPhysicalPlan for BroadcastSink { Box::new(std::iter::once(&self.input)) } - fn children_mut<'a>(&'a self) -> Box> + 'a> { + fn children_mut<'a>(&'a mut self) -> Box> + 'a> { Box::new(std::iter::once(&mut self.input)) } } -pub fn build_broadcast_plan(broadcast_id: u32) -> Result { - let broadcast_source = Box::new(PhysicalPlan::BroadcastSource(BroadcastSource { - plan_id: 0, +pub fn build_broadcast_plan(broadcast_id: u32) -> Result> { + let broadcast_source: Box = Box::new(BroadcastSource { + meta: PhysicalPlanMeta::new("BroadcastSource"), broadcast_id, - })); - let exchange = Box::new(PhysicalPlan::Exchange(Exchange { - plan_id: 0, + }); + + let exchange = Box::new(Exchange { input: broadcast_source, kind: FragmentKind::Expansive, keys: vec![], allow_adjust_parallelism: true, ignore_exchange: false, - })); - let broadcast_sink = PhysicalPlan::BroadcastSink(BroadcastSink { - plan_id: 0, + meta: PhysicalPlanMeta::new("Exchange"), + }); + + Ok(Box::new(BroadcastSink { broadcast_id, input: exchange, - }); - Ok(broadcast_sink) + meta: PhysicalPlanMeta::new("BroadcastSink"), + })) } -pub fn build_broadcast_plans(ctx: &dyn TableContext) -> Result> { +pub fn build_broadcast_plans(ctx: &dyn TableContext) -> Result>> { let mut plans = vec![]; let next_broadcast_id = ctx.get_next_broadcast_id(); ctx.reset_broadcast_id(); diff --git a/src/query/sql/src/executor/physical_plans/physical_cache_scan.rs b/src/query/sql/src/executor/physical_plans/physical_cache_scan.rs index 9bfa453473dc4..b112decffdf76 100644 --- a/src/query/sql/src/executor/physical_plans/physical_cache_scan.rs +++ b/src/query/sql/src/executor/physical_plans/physical_cache_scan.rs @@ -23,13 +23,12 @@ use crate::ColumnSet; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct CacheScan { - // A unique id of operator in a `PhysicalPlan` tree, only used for display. - pub plan_id: u32, - meta: PhysicalPlanMeta, + pub meta: PhysicalPlanMeta, pub cache_source: CacheSource, pub output_schema: DataSchemaRef, } +#[typetag::serde] impl IPhysicalPlan for CacheScan { fn get_meta(&self) -> &PhysicalPlanMeta { &self.meta @@ -49,7 +48,7 @@ impl PhysicalPlanBuilder { &mut self, scan: &crate::plans::CacheScan, required: ColumnSet, - ) -> Result { + ) -> Result> { // 1. Prune unused Columns. let used: ColumnSet = required.intersection(&scan.columns).cloned().collect(); let (cache_source, fields) = if used == scan.columns { @@ -62,9 +61,9 @@ impl PhysicalPlanBuilder { ) }; // 2. Build physical plan. - Ok(PhysicalPlan::CacheScan(CacheScan { - plan_id: 0, + Ok(Box::new(CacheScan { cache_source, + meta: PhysicalPlanMeta::new("CacheScan"), output_schema: DataSchemaRefExt::create(fields), })) } diff --git a/src/query/sql/src/executor/physical_plans/physical_column_mutation.rs b/src/query/sql/src/executor/physical_plans/physical_column_mutation.rs index c413055881e2f..1d5bb8b11adda 100644 --- a/src/query/sql/src/executor/physical_plans/physical_column_mutation.rs +++ b/src/query/sql/src/executor/physical_plans/physical_column_mutation.rs @@ -24,8 +24,7 @@ use databend_common_exception::Result; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct ColumnMutation { - pub plan_id: u32, - meta: PhysicalPlanMeta, + pub meta: PhysicalPlanMeta, pub input: Box, pub table_info: TableInfo, pub mutation_expr: Option>, @@ -37,6 +36,7 @@ pub struct ColumnMutation { pub table_meta_timestamps: TableMetaTimestamps, } +#[typetag::serde] impl IPhysicalPlan for ColumnMutation { fn get_meta(&self) -> &PhysicalPlanMeta { &self.meta @@ -54,7 +54,7 @@ impl IPhysicalPlan for ColumnMutation { Box::new(std::iter::once(&self.input)) } - fn children_mut<'a>(&'a self) -> Box> + 'a> { + fn children_mut<'a>(&'a mut self) -> Box> + 'a> { Box::new(std::iter::once(&mut self.input)) } } diff --git a/src/query/sql/src/executor/physical_plans/physical_commit_sink.rs b/src/query/sql/src/executor/physical_plans/physical_commit_sink.rs index 7d8b36e8ec628..bda53238a9565 100644 --- a/src/query/sql/src/executor/physical_plans/physical_commit_sink.rs +++ b/src/query/sql/src/executor/physical_plans/physical_commit_sink.rs @@ -30,8 +30,7 @@ use databend_common_exception::Result; /// The commit sink is used to commit the data to the table. #[derive(serde::Serialize, serde::Deserialize, Clone, Debug)] pub struct CommitSink { - pub plan_id: u32, - meta: PhysicalPlanMeta, + pub meta: PhysicalPlanMeta, pub input: Box, pub snapshot: Option>, pub table_info: TableInfo, @@ -44,6 +43,7 @@ pub struct CommitSink { pub recluster_info: Option, } +#[typetag::serde] impl IPhysicalPlan for CommitSink { fn get_meta(&self) -> &PhysicalPlanMeta { &self.meta @@ -61,7 +61,7 @@ impl IPhysicalPlan for CommitSink { Box::new(std::iter::once(&self.input)) } - fn children_mut<'a>(&'a self) -> Box> + 'a> { + fn children_mut<'a>(&'a mut self) -> Box> + 'a> { Box::new(std::iter::once(&mut self.input)) } } diff --git a/src/query/sql/src/executor/physical_plans/physical_compact_source.rs b/src/query/sql/src/executor/physical_plans/physical_compact_source.rs index 84594fbf70d03..909ff175dcb5b 100644 --- a/src/query/sql/src/executor/physical_plans/physical_compact_source.rs +++ b/src/query/sql/src/executor/physical_plans/physical_compact_source.rs @@ -33,14 +33,14 @@ use crate::executor::PhysicalPlanBuilder; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct CompactSource { - pub plan_id: u32, - meta: PhysicalPlanMeta, + pub meta: PhysicalPlanMeta, pub parts: Partitions, pub table_info: TableInfo, pub column_ids: HashSet, pub table_meta_timestamps: TableMetaTimestamps, } +#[typetag::serde] impl IPhysicalPlan for CompactSource { fn get_meta(&self) -> &PhysicalPlanMeta { &self.meta @@ -55,7 +55,7 @@ impl PhysicalPlanBuilder { pub async fn build_compact_block( &mut self, compact_block: &crate::plans::OptimizeCompactBlock, - ) -> Result { + ) -> Result> { let crate::plans::OptimizeCompactBlock { catalog, database, @@ -83,28 +83,28 @@ impl PhysicalPlanBuilder { .get_table_meta_timestamps(tbl.as_ref(), Some(snapshot.clone()))?; let merge_meta = parts.partitions_type() == PartInfoType::LazyLevel; - let mut root = PhysicalPlan::CompactSource(Box::new(CompactSource { + let mut root: Box = Box::new(CompactSource { parts, table_info: table_info.clone(), column_ids: snapshot.schema.to_leaf_column_id_set(), - plan_id: u32::MAX, table_meta_timestamps, - })); + meta: PhysicalPlanMeta::new("ConstantTableScan"), + }); let is_distributed = (!self.ctx.get_cluster().is_empty()) && self.ctx.get_settings().get_enable_distributed_compact()?; if is_distributed { - root = PhysicalPlan::Exchange(Exchange { - plan_id: 0, - input: Box::new(root), + root = Box::new(Exchange { + input: root, kind: FragmentKind::Merge, keys: vec![], allow_adjust_parallelism: true, ignore_exchange: false, + meta: PhysicalPlanMeta::new("ConstantTableScan"), }); } - root = PhysicalPlan::CommitSink(Box::new(CommitSink { + root = Box::new(CommitSink { input: Box::new(root), table_info, snapshot: Some(snapshot), @@ -114,10 +114,10 @@ impl PhysicalPlanBuilder { }, update_stream_meta: vec![], deduplicated_label: None, - plan_id: u32::MAX, recluster_info: None, table_meta_timestamps, - })); + meta: PhysicalPlanMeta::new("CommitSink"), + }); root.adjust_plan_id(&mut 0); Ok(root) diff --git a/src/query/sql/src/executor/physical_plans/physical_constant_table_scan.rs b/src/query/sql/src/executor/physical_plans/physical_constant_table_scan.rs index cd1e88aadd9e9..db74487e20cc8 100644 --- a/src/query/sql/src/executor/physical_plans/physical_constant_table_scan.rs +++ b/src/query/sql/src/executor/physical_plans/physical_constant_table_scan.rs @@ -23,14 +23,13 @@ use crate::IndexType; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct ConstantTableScan { - // A unique id of operator in a `PhysicalPlan` tree, only used for display. - pub plan_id: u32, meta: PhysicalPlanMeta, pub values: Vec, pub num_rows: usize, pub output_schema: DataSchemaRef, } +#[typetag::serde] impl IPhysicalPlan for ConstantTableScan { fn get_meta(&self) -> &PhysicalPlanMeta { &self.meta @@ -60,7 +59,7 @@ impl PhysicalPlanBuilder { &mut self, scan: &crate::plans::ConstantTableScan, required: ColumnSet, - ) -> Result { + ) -> Result> { debug_assert!(scan .schema .fields @@ -77,19 +76,19 @@ impl PhysicalPlanBuilder { schema, .. } = scan.prune_columns(used); - return Ok(PhysicalPlan::ConstantTableScan(ConstantTableScan { - plan_id: 0, + return Ok(Box::new(ConstantTableScan { values, num_rows, output_schema: schema, + meta: PhysicalPlanMeta::new("ConstantTableScan"), })); } - Ok(PhysicalPlan::ConstantTableScan(ConstantTableScan { - plan_id: 0, + Ok(Box::new(ConstantTableScan { values: scan.values.clone(), num_rows: scan.num_rows, output_schema: scan.schema.clone(), + meta: PhysicalPlanMeta::new("ConstantTableScan"), })) } } diff --git a/src/query/sql/src/executor/physical_plans/physical_copy_into_location.rs b/src/query/sql/src/executor/physical_plans/physical_copy_into_location.rs index 4aabfe4300db2..4eff317c1283e 100644 --- a/src/query/sql/src/executor/physical_plans/physical_copy_into_location.rs +++ b/src/query/sql/src/executor/physical_plans/physical_copy_into_location.rs @@ -27,8 +27,7 @@ use crate::ColumnBinding; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct CopyIntoLocation { - pub plan_id: u32, - meta: PhysicalPlanMeta, + pub meta: PhysicalPlanMeta, pub input: Box, pub project_columns: Vec, pub input_data_schema: DataSchemaRef, @@ -36,6 +35,7 @@ pub struct CopyIntoLocation { pub info: CopyIntoLocationInfo, } +#[typetag::serde] impl IPhysicalPlan for CopyIntoLocation { fn get_meta(&self) -> &PhysicalPlanMeta { &self.meta @@ -57,7 +57,7 @@ impl IPhysicalPlan for CopyIntoLocation { Box::new(std::iter::once(&self.input)) } - fn children_mut<'a>(&'a self) -> Box> + 'a> { + fn children_mut<'a>(&'a mut self) -> Box> + 'a> { Box::new(std::iter::once(&mut self.input)) } diff --git a/src/query/sql/src/executor/physical_plans/physical_copy_into_table.rs b/src/query/sql/src/executor/physical_plans/physical_copy_into_table.rs index a3ea57acf6511..14737fed539b2 100644 --- a/src/query/sql/src/executor/physical_plans/physical_copy_into_table.rs +++ b/src/query/sql/src/executor/physical_plans/physical_copy_into_table.rs @@ -29,9 +29,7 @@ use crate::executor::{IPhysicalPlan, PhysicalPlanMeta}; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct CopyIntoTable { - pub plan_id: u32, - - meta: PhysicalPlanMeta, + pub meta: PhysicalPlanMeta, pub required_values_schema: DataSchemaRef, pub values_consts: Vec, pub required_source_schema: DataSchemaRef, @@ -46,6 +44,7 @@ pub struct CopyIntoTable { pub table_meta_timestamps: TableMetaTimestamps, } +#[typetag::serde] impl IPhysicalPlan for CopyIntoTable { fn get_meta(&self) -> &PhysicalPlanMeta { &self.meta @@ -66,7 +65,7 @@ impl IPhysicalPlan for CopyIntoTable { } } - fn children_mut<'a>(&'a self) -> Box> + 'a> { + fn children_mut<'a>(&'a mut self) -> Box> + 'a> { match &mut self.source { CopyIntoTableSource::Query(v) => Box::new(std::iter::once(v)), CopyIntoTableSource::Stage(v) => Box::new(std::iter::once(v)), diff --git a/src/query/sql/src/executor/physical_plans/physical_distributed_insert_select.rs b/src/query/sql/src/executor/physical_plans/physical_distributed_insert_select.rs index 001a4cee9e685..e9958d080b5ef 100644 --- a/src/query/sql/src/executor/physical_plans/physical_distributed_insert_select.rs +++ b/src/query/sql/src/executor/physical_plans/physical_distributed_insert_select.rs @@ -23,10 +23,7 @@ use databend_common_exception::Result; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct DistributedInsertSelect { - /// A unique id of operator in a `PhysicalPlan` tree. - pub plan_id: u32, - - meta: PhysicalPlanMeta, + pub meta: PhysicalPlanMeta, pub input: Box, pub table_info: TableInfo, pub insert_schema: DataSchemaRef, @@ -36,6 +33,7 @@ pub struct DistributedInsertSelect { pub table_meta_timestamps: TableMetaTimestamps, } +#[typetag::serde] impl IPhysicalPlan for DistributedInsertSelect { fn get_meta(&self) -> &PhysicalPlanMeta { &self.meta @@ -53,7 +51,7 @@ impl IPhysicalPlan for DistributedInsertSelect { Box::new(std::iter::once(&self.input)) } - fn children_mut<'a>(&'a self) -> Box> + 'a> { + fn children_mut<'a>(&'a mut self) -> Box> + 'a> { Box::new(std::iter::once(&mut self.input)) } diff --git a/src/query/sql/src/executor/physical_plans/physical_eval_scalar.rs b/src/query/sql/src/executor/physical_plans/physical_eval_scalar.rs index 22efac099b159..6e1c7942339cc 100644 --- a/src/query/sql/src/executor/physical_plans/physical_eval_scalar.rs +++ b/src/query/sql/src/executor/physical_plans/physical_eval_scalar.rs @@ -12,9 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::collections::BTreeSet; +use std::collections::{BTreeSet, HashMap}; use std::collections::HashSet; use std::sync::Arc; +use itertools::Itertools; use databend_common_catalog::plan::DataSourcePlan; use databend_common_exception::Result; use databend_common_expression::ConstantFolder; @@ -46,8 +47,6 @@ use crate::TypeCheck; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct EvalScalar { meta: PhysicalPlanMeta, - // A unique id of operator in a `PhysicalPlan` tree, only used for display. - pub plan_id: u32, pub projections: ColumnSet, pub input: Box, pub exprs: Vec<(RemoteExpr, IndexType)>, @@ -56,6 +55,7 @@ pub struct EvalScalar { pub stat_info: Option, } +#[typetag::serde] impl IPhysicalPlan for EvalScalar { fn get_meta(&self) -> &PhysicalPlanMeta { &self.meta @@ -93,7 +93,7 @@ impl IPhysicalPlan for EvalScalar { Box::new(std::iter::once(&self.input)) } - fn children_mut<'a>(&'a self) -> Box> + 'a> { + fn children_mut<'a>(&'a mut self) -> Box> + 'a> { Box::new(std::iter::once(&mut self.input)) } @@ -101,6 +101,27 @@ impl IPhysicalPlan for EvalScalar { fn try_find_single_data_source(&self) -> Option<&DataSourcePlan> { self.input.try_find_single_data_source() } + + fn get_desc(&self) -> Result { + Ok(self + .exprs + .iter() + .map(|(x, _)| x.as_expr(&BUILTIN_FUNCTIONS).sql_display()) + .join(", ") + ) + } + + fn get_labels(&self) -> Result>> { + Ok(HashMap::from([ + ( + String::from("List of Expressions"), + self.exprs + .iter() + .map(|(x, _)| x.as_expr(&BUILTIN_FUNCTIONS).sql_display()) + .collect(), + ) + ])) + } } impl PhysicalPlanBuilder { @@ -110,7 +131,7 @@ impl PhysicalPlanBuilder { eval_scalar: &crate::plans::EvalScalar, mut required: ColumnSet, stat_info: PlanStatsInfo, - ) -> Result { + ) -> Result> { // 1. Prune unused Columns. let column_projections = required.clone(); let mut used = vec![]; @@ -149,9 +170,9 @@ impl PhysicalPlanBuilder { &mut self, eval_scalar: &crate::plans::EvalScalar, column_projections: Vec, - input: PhysicalPlan, + input: Box, stat_info: PlanStatsInfo, - ) -> Result { + ) -> Result> { let input_schema = input.output_schema()?; let exprs = eval_scalar .items @@ -188,12 +209,12 @@ impl PhysicalPlanBuilder { projections.insert(index + input_column_nums); } } - Ok(PhysicalPlan::EvalScalar(EvalScalar { - plan_id: 0, - projections, - input: Box::new(input), + Ok(Box::new(EvalScalar { + input, exprs, + projections, stat_info: Some(stat_info), + meta: PhysicalPlanMeta::new("EvalScalar"), })) } diff --git a/src/query/sql/src/executor/physical_plans/physical_exchange.rs b/src/query/sql/src/executor/physical_plans/physical_exchange.rs index 68a5152f066c8..da5bbba5ccd7a 100644 --- a/src/query/sql/src/executor/physical_plans/physical_exchange.rs +++ b/src/query/sql/src/executor/physical_plans/physical_exchange.rs @@ -28,10 +28,7 @@ use crate::TypeCheck; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct Exchange { - // A unique id of operator in a `PhysicalPlan` tree, only used for display. - pub plan_id: u32, - - meta: PhysicalPlanMeta, + pub meta: PhysicalPlanMeta, pub input: Box, pub kind: FragmentKind, pub keys: Vec, @@ -39,6 +36,7 @@ pub struct Exchange { pub allow_adjust_parallelism: bool, } +#[typetag::serde] impl IPhysicalPlan for Exchange { fn get_meta(&self) -> &PhysicalPlanMeta { &self.meta @@ -52,7 +50,7 @@ impl IPhysicalPlan for Exchange { Box::new(std::iter::once(&self.input)) } - fn children_mut<'a>(&'a self) -> Box> + 'a> { + fn children_mut<'a>(&'a mut self) -> Box> + 'a> { Box::new(std::iter::once(&mut self.input)) } @@ -72,7 +70,7 @@ impl PhysicalPlanBuilder { s_expr: &SExpr, exchange: &crate::plans::Exchange, mut required: ColumnSet, - ) -> Result { + ) -> Result> { // 1. Prune unused Columns. if let crate::plans::Exchange::Hash(exprs) = exchange { for expr in exprs { @@ -105,13 +103,13 @@ impl PhysicalPlanBuilder { FragmentKind::Merge } }; - Ok(PhysicalPlan::Exchange(Exchange { - plan_id: 0, + Ok(Box::new(Exchange { input, kind, keys, allow_adjust_parallelism, ignore_exchange: false, + meta: PhysicalPlanMeta::new("Exchange"), })) } } diff --git a/src/query/sql/src/executor/physical_plans/physical_exchange_sink.rs b/src/query/sql/src/executor/physical_plans/physical_exchange_sink.rs index d0090caf3def7..f09fea69b7d5f 100644 --- a/src/query/sql/src/executor/physical_plans/physical_exchange_sink.rs +++ b/src/query/sql/src/executor/physical_plans/physical_exchange_sink.rs @@ -22,9 +22,7 @@ use crate::executor::{IPhysicalPlan, PhysicalPlan, PhysicalPlanMeta}; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct ExchangeSink { - // A unique id of operator in a `PhysicalPlan` tree, only used for display. - pub plan_id: u32, - meta: PhysicalPlanMeta, + pub meta: PhysicalPlanMeta, pub input: Box, // Input schema of exchanged data pub schema: DataSchemaRef, @@ -40,6 +38,7 @@ pub struct ExchangeSink { pub allow_adjust_parallelism: bool, } +#[typetag::serde] impl IPhysicalPlan for ExchangeSink { fn get_meta(&self) -> &PhysicalPlanMeta { &self.meta @@ -57,7 +56,7 @@ impl IPhysicalPlan for ExchangeSink { Box::new(std::iter::once(&self.input)) } - fn children_mut<'a>(&'a self) -> Box> + 'a> { + fn children_mut<'a>(&'a mut self) -> Box> + 'a> { Box::new(std::iter::once(&mut self.input)) } diff --git a/src/query/sql/src/executor/physical_plans/physical_exchange_source.rs b/src/query/sql/src/executor/physical_plans/physical_exchange_source.rs index 2b56dfa1e967b..0f2c2fed4eb41 100644 --- a/src/query/sql/src/executor/physical_plans/physical_exchange_source.rs +++ b/src/query/sql/src/executor/physical_plans/physical_exchange_source.rs @@ -19,9 +19,7 @@ use crate::executor::{IPhysicalPlan, PhysicalPlanMeta}; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct ExchangeSource { - // A unique id of operator in a `PhysicalPlan` tree, only used for display. - pub plan_id: u32, - meta: PhysicalPlanMeta, + pub meta: PhysicalPlanMeta, // Output schema of exchanged data pub schema: DataSchemaRef, @@ -31,6 +29,7 @@ pub struct ExchangeSource { pub query_id: String, } +#[typetag::serde] impl IPhysicalPlan for ExchangeSource { fn get_meta(&self) -> &PhysicalPlanMeta { &self.meta diff --git a/src/query/sql/src/executor/physical_plans/physical_expression_scan.rs b/src/query/sql/src/executor/physical_plans/physical_expression_scan.rs index 5d9026bef5e49..ee83c8540c555 100644 --- a/src/query/sql/src/executor/physical_plans/physical_expression_scan.rs +++ b/src/query/sql/src/executor/physical_plans/physical_expression_scan.rs @@ -26,14 +26,13 @@ use crate::TypeCheck; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct ExpressionScan { - // A unique id of operator in a `PhysicalPlan` tree, only used for display. - pub plan_id: u32, - meta: PhysicalPlanMeta, + pub meta: PhysicalPlanMeta, pub values: Vec>, pub input: Box, pub output_schema: DataSchemaRef, } +#[typetag::serde] impl IPhysicalPlan for ExpressionScan { fn get_meta(&self) -> &PhysicalPlanMeta { &self.meta @@ -51,7 +50,7 @@ impl IPhysicalPlan for ExpressionScan { Box::new(std::iter::once(&self.input)) } - fn children_mut<'a>(&'a self) -> Box> + 'a> { + fn children_mut<'a>(&'a mut self) -> Box> + 'a> { Box::new(std::iter::once(&mut self.input)) } } @@ -62,7 +61,7 @@ impl PhysicalPlanBuilder { s_expr: &SExpr, scan: &crate::plans::ExpressionScan, required: ColumnSet, - ) -> Result { + ) -> Result> { let input = self.build(s_expr.child(0)?, required).await?; let input_schema = input.output_schema()?; @@ -85,11 +84,11 @@ impl PhysicalPlanBuilder { }) .collect::>>()?; - Ok(PhysicalPlan::ExpressionScan(ExpressionScan { - plan_id: 0, + Ok(Box::new(ExpressionScan { values, input: Box::new(input), output_schema: scan.schema.clone(), + meta: PhysicalPlanMeta::new("ExpressionScan"), })) } } diff --git a/src/query/sql/src/executor/physical_plans/physical_filter.rs b/src/query/sql/src/executor/physical_plans/physical_filter.rs index b0325b9b12292..96e3ab61b35d5 100644 --- a/src/query/sql/src/executor/physical_plans/physical_filter.rs +++ b/src/query/sql/src/executor/physical_plans/physical_filter.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::collections::HashMap; use databend_common_catalog::plan::DataSourcePlan; use databend_common_exception::Result; use databend_common_expression::ConstantFolder; @@ -31,8 +32,6 @@ use crate::TypeCheck; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct Filter { meta: PhysicalPlanMeta, - // A unique id of operator in a `PhysicalPlan` tree, only used for display. - pub plan_id: u32, pub projections: ColumnSet, pub input: Box, // Assumption: expression's data type must be `DataType::Boolean`. @@ -42,6 +41,7 @@ pub struct Filter { pub stat_info: Option, } +#[typetag::serde] impl IPhysicalPlan for Filter { fn get_meta(&self) -> &PhysicalPlanMeta { &self.meta @@ -51,14 +51,6 @@ impl IPhysicalPlan for Filter { &mut self.meta } - fn children<'a>(&'a self) -> Box> + 'a> { - Box::new(std::iter::once(&self.input)) - } - - fn children_mut<'a>(&'a self) -> Box> + 'a> { - Box::new(std::iter::once(&mut self.input)) - } - fn output_schema(&self) -> Result { let input_schema = self.input.output_schema()?; let mut fields = Vec::with_capacity(self.projections.len()); @@ -70,10 +62,37 @@ impl IPhysicalPlan for Filter { Ok(DataSchemaRefExt::create(fields)) } + fn children<'a>(&'a self) -> Box> + 'a> { + Box::new(std::iter::once(&self.input)) + } + + fn children_mut<'a>(&'a mut self) -> Box> + 'a> { + Box::new(std::iter::once(&mut self.input)) + } + #[recursive::recursive] fn try_find_single_data_source(&self) -> Option<&DataSourcePlan> { self.input.try_find_single_data_source() } + + fn get_desc(&self) -> Result { + Ok(match self.predicates.is_empty() { + true => String::new(), + false => self.predicates[0].as_expr(&BUILTIN_FUNCTIONS).sql_display(), + }) + } + + fn get_labels(&self) -> Result>> { + Ok(HashMap::from([ + ( + String::from("Filter condition"), + self.predicates + .iter() + .map(|x| x.as_expr(&BUILTIN_FUNCTIONS).sql_display()) + .collect(), + ) + ])) + } } impl PhysicalPlanBuilder { @@ -83,7 +102,7 @@ impl PhysicalPlanBuilder { filter: &crate::plans::Filter, mut required: ColumnSet, stat_info: PlanStatsInfo, - ) -> Result { + ) -> Result> { // 1. Prune unused Columns. let used = filter.predicates.iter().fold(required.clone(), |acc, v| { acc.union(&v.used_columns()).cloned().collect() @@ -104,8 +123,8 @@ impl PhysicalPlanBuilder { } } - Ok(PhysicalPlan::Filter(Filter { - plan_id: 0, + Ok(Box::new(Filter { + meta: PhysicalPlanMeta::new("Filter"), projections, input, predicates: filter diff --git a/src/query/sql/src/executor/physical_plans/physical_hash_join.rs b/src/query/sql/src/executor/physical_plans/physical_hash_join.rs index cd57535f66577..ca9a5ee39fc88 100644 --- a/src/query/sql/src/executor/physical_plans/physical_hash_join.rs +++ b/src/query/sql/src/executor/physical_plans/physical_hash_join.rs @@ -66,9 +66,7 @@ type MergedFieldsResult = ( #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct HashJoin { - // A unique id of operator in a `PhysicalPlan` tree, only used for display. - pub plan_id: u32, - meta: PhysicalPlanMeta, + pub meta: PhysicalPlanMeta, // After building the probe key and build key, we apply probe_projections to probe_datablock // and build_projections to build_datablock, which can help us reduce memory usage and calls // of expensive functions (take_compacted_indices and gather), after processing other_conditions, @@ -109,6 +107,7 @@ pub struct HashJoin { pub broadcast_id: Option, } +#[typetag::serde] impl IPhysicalPlan for HashJoin { fn get_meta(&self) -> &PhysicalPlanMeta { &self.meta @@ -126,9 +125,69 @@ impl IPhysicalPlan for HashJoin { Box::new(std::iter::once(&self.probe).chain(std::iter::once(&self.build))) } - fn children_mut<'a>(&'a self) -> Box> + 'a> { + fn children_mut<'a>(&'a mut self) -> Box> + 'a> { Box::new(std::iter::once(&mut self.probe).chain(std::iter::once(&mut self.build))) } + + fn get_desc(&self) -> Result { + let mut conditions = self + .build_keys + .iter() + .zip(self.probe_keys.iter()) + .map(|(l, r)| { + format!( + "({} = {})", + l.as_expr(&BUILTIN_FUNCTIONS).sql_display(), + r.as_expr(&BUILTIN_FUNCTIONS).sql_display() + ) + }) + .collect::>(); + + conditions.extend( + self.non_equi_conditions + .iter() + .map(|x| x.as_expr(&BUILTIN_FUNCTIONS).sql_display()), + ); + + Ok(conditions.join(" AND ")) + } + + fn get_labels(&self) -> Result>> { + let mut labels = HashMap::with_capacity(4); + labels.insert(String::from("Join Type"), vec![self.join_type.to_string()]); + + if !self.build_keys.is_empty() { + labels.insert( + String::from("Join Build Side Keys"), + self.build_keys + .iter() + .map(|x| x.as_expr(&BUILTIN_FUNCTIONS).sql_display()) + .collect(), + ); + } + + if !self.probe_keys.is_empty() { + labels.insert( + String::from("Join Probe Side Keys"), + self.probe_keys + .iter() + .map(|x| x.as_expr(&BUILTIN_FUNCTIONS).sql_display()) + .collect(), + ); + } + + if !self.non_equi_conditions.is_empty() { + labels.insert( + String::from("Join Conditions"), + self.non_equi_conditions + .iter() + .map(|x| x.as_expr(&BUILTIN_FUNCTIONS).sql_display()) + .collect(), + ); + } + + Ok(labels) + } } impl PhysicalPlanBuilder { @@ -138,7 +197,7 @@ impl PhysicalPlanBuilder { s_expr: &SExpr, left_required: ColumnSet, right_required: ColumnSet, - ) -> Result<(Box, Box)> { + ) -> Result<(Box, Box)> { let probe_side = Box::new(self.build(s_expr.child(0)?, left_required).await?); let build_side = Box::new(self.build(s_expr.child(1)?, right_required).await?); @@ -172,7 +231,7 @@ impl PhysicalPlanBuilder { pub(crate) fn prepare_build_schema( &self, join_type: &JoinType, - build_side: &PhysicalPlan, + build_side: &Box, ) -> Result { match join_type { JoinType::Left | JoinType::LeftSingle | JoinType::LeftAsof | JoinType::Full => { @@ -208,7 +267,7 @@ impl PhysicalPlanBuilder { pub(crate) fn prepare_probe_schema( &self, join_type: &JoinType, - probe_side: &PhysicalPlan, + probe_side: &Box, ) -> Result { match join_type { JoinType::Right | JoinType::RightSingle | JoinType::RightAsof | JoinType::Full => { @@ -236,8 +295,8 @@ impl PhysicalPlanBuilder { /// * `build_side` - The build side physical plan fn unify_keys( &self, - probe_side: &mut Box, - build_side: &mut Box, + probe_side: &mut Box, + build_side: &mut Box, ) -> Result<()> { // Unify the data types of the left and right exchange keys if let ( @@ -807,8 +866,8 @@ impl PhysicalPlanBuilder { &self, s_expr: &SExpr, join: &Join, - probe_side: Box, - build_side: Box, + probe_side: Box, + build_side: Box, projections: ColumnSet, probe_projections: ColumnSet, build_projections: ColumnSet, @@ -821,7 +880,7 @@ impl PhysicalPlanBuilder { build_side_cache_info: Option<(usize, HashMap)>, runtime_filter: PhysicalRuntimeFilters, stat_info: PlanStatsInfo, - ) -> Result { + ) -> Result> { let build_side_data_distribution = s_expr.build_side_child().get_data_distribution()?; let broadcast_id = if build_side_data_distribution .as_ref() @@ -831,8 +890,7 @@ impl PhysicalPlanBuilder { } else { None }; - Ok(PhysicalPlan::HashJoin(HashJoin { - plan_id: 0, + Ok(Box::new(HashJoin { projections, build_projections, probe_projections, @@ -844,6 +902,7 @@ impl PhysicalPlanBuilder { is_null_equal, non_equi_conditions, marker_index: join.marker_index, + meta: PhysicalPlanMeta::new("HashJoin"), from_correlated_subquery: join.from_correlated_subquery, probe_to_build, output_schema, @@ -865,7 +924,7 @@ impl PhysicalPlanBuilder { left_required: ColumnSet, right_required: ColumnSet, stat_info: PlanStatsInfo, - ) -> Result { + ) -> Result> { // Step 1: Build probe and build sides let (mut probe_side, mut build_side) = self .build_join_sides(s_expr, left_required, right_required) diff --git a/src/query/sql/src/executor/physical_plans/physical_join.rs b/src/query/sql/src/executor/physical_plans/physical_join.rs index 1b09ecc6a2eed..573c7eb139665 100644 --- a/src/query/sql/src/executor/physical_plans/physical_join.rs +++ b/src/query/sql/src/executor/physical_plans/physical_join.rs @@ -16,7 +16,7 @@ use databend_common_exception::Result; use crate::binder::JoinPredicate; use crate::executor::explain::PlanStatsInfo; -use crate::executor::PhysicalPlan; +use crate::executor::{IPhysicalPlan, PhysicalPlan}; use crate::executor::PhysicalPlanBuilder; use crate::optimizer::ir::RelExpr; use crate::optimizer::ir::RelationalProperty; @@ -131,7 +131,7 @@ impl PhysicalPlanBuilder { join: &crate::plans::Join, required: ColumnSet, stat_info: PlanStatsInfo, - ) -> Result { + ) -> Result> { // 1. Prune unused Columns. let mut others_required = join .non_equi_conditions diff --git a/src/query/sql/src/executor/physical_plans/physical_limit.rs b/src/query/sql/src/executor/physical_plans/physical_limit.rs index 4ed509b86084f..69d81536eaa78 100644 --- a/src/query/sql/src/executor/physical_plans/physical_limit.rs +++ b/src/query/sql/src/executor/physical_plans/physical_limit.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::collections::HashMap; use databend_common_catalog::plan::DataSourcePlan; use databend_common_exception::ErrorCode; use databend_common_exception::Result; @@ -29,8 +30,6 @@ use crate::ColumnSet; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct Limit { - // A unique id of operator in a `PhysicalPlan` tree, only used for display. - pub plan_id: u32, meta: PhysicalPlanMeta, pub input: Box, pub limit: Option, @@ -40,6 +39,7 @@ pub struct Limit { pub stat_info: Option, } +#[typetag::serde] impl IPhysicalPlan for Limit { fn get_meta(&self) -> &PhysicalPlanMeta { &self.meta @@ -53,7 +53,7 @@ impl IPhysicalPlan for Limit { Box::new(std::iter::once(&self.input)) } - fn children_mut<'a>(&'a self) -> Box> + 'a> { + fn children_mut<'a>(&'a mut self) -> Box> + 'a> { Box::new(std::iter::once(&mut self.input)) } @@ -61,6 +61,24 @@ impl IPhysicalPlan for Limit { fn try_find_single_data_source(&self) -> Option<&DataSourcePlan> { self.input.try_find_single_data_source() } + + fn get_desc(&self) -> Result { + Ok(match self.limit { + Some(limit) => format!("LIMIT {} OFFSET {}", limit, self.offset), + None => format!("OFFSET {}", self.offset), + }) + } + + fn get_labels(&self) -> Result>> { + let mut labels = HashMap::with_capacity(2); + labels.insert(String::from("Offset"), vec![self.offset.to_string()]); + + if let Some(limit) = self.limit { + labels.insert(String::from("Number of rows"), vec![limit.to_string()]); + } + + Ok(labels) + } } impl PhysicalPlanBuilder { @@ -70,7 +88,7 @@ impl PhysicalPlanBuilder { limit: &crate::plans::Limit, mut required: ColumnSet, stat_info: PlanStatsInfo, - ) -> Result { + ) -> Result> { // 1. Prune unused Columns. // Apply lazy. let metadata = self.metadata.read().clone(); @@ -85,12 +103,12 @@ impl PhysicalPlanBuilder { let input_plan = self.build(s_expr.child(0)?, required).await?; let metadata = self.metadata.read().clone(); if limit.before_exchange || metadata.lazy_columns().is_empty() { - return Ok(PhysicalPlan::Limit(Limit { - plan_id: 0, + return Ok(Box::new(Limit { input: Box::new(input_plan), limit: limit.limit, offset: limit.offset, stat_info: Some(stat_info), + meta: PhysicalPlanMeta::new("Limit"), })); } @@ -105,12 +123,12 @@ impl PhysicalPlanBuilder { .ok_or_else(|| ErrorCode::Internal("Internal column _row_id is not found"))?; if !input_schema.has_field(&row_id_col_index.to_string()) { - return Ok(PhysicalPlan::Limit(Limit { - plan_id: 0, + return Ok(Box::new(Limit { input: Box::new(input_plan), limit: limit.limit, offset: limit.offset, stat_info: Some(stat_info), + meta: PhysicalPlanMeta::new("Limit"), })); } @@ -128,12 +146,12 @@ impl PhysicalPlanBuilder { if limit.before_exchange || lazy_columns.is_empty() { // If there is no lazy column, we don't need to build a `RowFetch` plan. - return Ok(PhysicalPlan::Limit(Limit { - plan_id: 0, + return Ok(Box::new(Limit { input: Box::new(input_plan), limit: limit.limit, offset: limit.offset, stat_info: Some(stat_info), + meta: PhysicalPlanMeta::new("Limit"), })); } @@ -165,15 +183,15 @@ impl PhysicalPlanBuilder { false, ); - Ok(PhysicalPlan::RowFetch(RowFetch { - plan_id: 0, - input: Box::new(PhysicalPlan::Limit(Limit { - plan_id: 0, + Ok(Box::new(RowFetch { + meta: PhysicalPlanMeta::new("RowFetch"), + input: Box::new(Limit { + meta: PhysicalPlanMeta::new("RowFetch"), input: Box::new(input_plan), limit: limit.limit, offset: limit.offset, stat_info: Some(stat_info.clone()), - })), + }), source: Box::new(source_info), row_id_col_offset, cols_to_fetch, diff --git a/src/query/sql/src/executor/physical_plans/physical_multi_table_insert.rs b/src/query/sql/src/executor/physical_plans/physical_multi_table_insert.rs index f644ebf57b01f..cd4ef71dfec15 100644 --- a/src/query/sql/src/executor/physical_plans/physical_multi_table_insert.rs +++ b/src/query/sql/src/executor/physical_plans/physical_multi_table_insert.rs @@ -27,12 +27,12 @@ use crate::executor::{IPhysicalPlan, PhysicalPlan, PhysicalPlanMeta}; use crate::ColumnSet; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct Duplicate { - pub plan_id: u32, - meta: PhysicalPlanMeta, + pub meta: PhysicalPlanMeta, pub input: Box, pub n: usize, } +#[typetag::serde] impl IPhysicalPlan for Duplicate { fn get_meta(&self) -> &PhysicalPlanMeta { &self.meta @@ -46,19 +46,19 @@ impl IPhysicalPlan for Duplicate { Box::new(std::iter::once(&self.input)) } - fn children_mut<'a>(&'a self) -> Box> + 'a> { + fn children_mut<'a>(&'a mut self) -> Box> + 'a> { Box::new(std::iter::once(&mut self.input)) } } #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct Shuffle { - pub plan_id: u32, - meta: PhysicalPlanMeta, + pub meta: PhysicalPlanMeta, pub input: Box, pub strategy: ShuffleStrategy, } +#[typetag::serde] impl IPhysicalPlan for Shuffle { fn get_meta(&self) -> &PhysicalPlanMeta { &self.meta @@ -72,7 +72,7 @@ impl IPhysicalPlan for Shuffle { Box::new(std::iter::once(&self.input)) } - fn children_mut<'a>(&'a self) -> Box> + 'a> { + fn children_mut<'a>(&'a mut self) -> Box> + 'a> { Box::new(std::iter::once(&mut self.input)) } } @@ -106,12 +106,12 @@ impl ShuffleStrategy { #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct ChunkFilter { - pub plan_id: u32, - meta: PhysicalPlanMeta, + pub meta: PhysicalPlanMeta, pub input: Box, pub predicates: Vec>, } +#[typetag::serde] impl IPhysicalPlan for ChunkFilter { fn get_meta(&self) -> &PhysicalPlanMeta { &self.meta @@ -125,19 +125,19 @@ impl IPhysicalPlan for ChunkFilter { Box::new(std::iter::once(&self.input)) } - fn children_mut<'a>(&'a self) -> Box> + 'a> { + fn children_mut<'a>(&'a mut self) -> Box> + 'a> { Box::new(std::iter::once(&mut self.input)) } } #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct ChunkEvalScalar { - pub plan_id: u32, - meta: PhysicalPlanMeta, + pub meta: PhysicalPlanMeta, pub input: Box, pub eval_scalars: Vec>, } +#[typetag::serde] impl IPhysicalPlan for ChunkEvalScalar { fn get_meta(&self) -> &PhysicalPlanMeta { &self.meta @@ -151,7 +151,7 @@ impl IPhysicalPlan for ChunkEvalScalar { Box::new(std::iter::once(&self.input)) } - fn children_mut<'a>(&'a self) -> Box> + 'a> { + fn children_mut<'a>(&'a mut self) -> Box> + 'a> { Box::new(std::iter::once(&mut self.input)) } } @@ -164,12 +164,12 @@ pub struct MultiInsertEvalScalar { #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct ChunkCastSchema { - pub plan_id: u32, - meta: PhysicalPlanMeta, + pub meta: PhysicalPlanMeta, pub input: Box, pub cast_schemas: Vec>, } +#[typetag::serde] impl IPhysicalPlan for ChunkCastSchema { fn get_meta(&self) -> &PhysicalPlanMeta { &self.meta @@ -183,7 +183,7 @@ impl IPhysicalPlan for ChunkCastSchema { Box::new(std::iter::once(&self.input)) } - fn children_mut<'a>(&'a self) -> Box> + 'a> { + fn children_mut<'a>(&'a mut self) -> Box> + 'a> { Box::new(std::iter::once(&mut self.input)) } } @@ -196,12 +196,12 @@ pub struct CastSchema { #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct ChunkFillAndReorder { - pub plan_id: u32, - meta: PhysicalPlanMeta, + pub meta: PhysicalPlanMeta, pub input: Box, pub fill_and_reorders: Vec>, } +#[typetag::serde] impl IPhysicalPlan for ChunkFillAndReorder { fn get_meta(&self) -> &PhysicalPlanMeta { &self.meta @@ -215,7 +215,7 @@ impl IPhysicalPlan for ChunkFillAndReorder { Box::new(std::iter::once(&self.input)) } - fn children_mut<'a>(&'a self) -> Box> + 'a> { + fn children_mut<'a>(&'a mut self) -> Box> + 'a> { Box::new(std::iter::once(&mut self.input)) } } @@ -228,12 +228,12 @@ pub struct FillAndReorder { #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct ChunkAppendData { - pub plan_id: u32, - meta: PhysicalPlanMeta, + pub meta: PhysicalPlanMeta, pub input: Box, pub target_tables: Vec, } +#[typetag::serde] impl IPhysicalPlan for ChunkAppendData { fn get_meta(&self) -> &PhysicalPlanMeta { &self.meta @@ -247,7 +247,7 @@ impl IPhysicalPlan for ChunkAppendData { Box::new(std::iter::once(&self.input)) } - fn children_mut<'a>(&'a self) -> Box> + 'a> { + fn children_mut<'a>(&'a mut self) -> Box> + 'a> { Box::new(std::iter::once(&mut self.input)) } } @@ -261,12 +261,12 @@ pub struct SerializableTable { #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct ChunkMerge { - pub plan_id: u32, - meta: PhysicalPlanMeta, + pub meta: PhysicalPlanMeta, pub input: Box, pub group_ids: Vec, } +#[typetag::serde] impl IPhysicalPlan for ChunkMerge { fn get_meta(&self) -> &PhysicalPlanMeta { &self.meta @@ -280,15 +280,14 @@ impl IPhysicalPlan for ChunkMerge { Box::new(std::iter::once(&self.input)) } - fn children_mut<'a>(&'a self) -> Box> + 'a> { + fn children_mut<'a>(&'a mut self) -> Box> + 'a> { Box::new(std::iter::once(&mut self.input)) } } #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct ChunkCommitInsert { - pub plan_id: u32, - meta: PhysicalPlanMeta, + pub meta: PhysicalPlanMeta, pub input: Box, pub update_stream_meta: Vec, pub overwrite: bool, @@ -296,6 +295,7 @@ pub struct ChunkCommitInsert { pub targets: Vec, } +#[typetag::serde] impl IPhysicalPlan for ChunkCommitInsert { fn get_meta(&self) -> &PhysicalPlanMeta { &self.meta @@ -309,7 +309,7 @@ impl IPhysicalPlan for ChunkCommitInsert { Box::new(std::iter::once(&self.input)) } - fn children_mut<'a>(&'a self) -> Box> + 'a> { + fn children_mut<'a>(&'a mut self) -> Box> + 'a> { Box::new(std::iter::once(&mut self.input)) } } diff --git a/src/query/sql/src/executor/physical_plans/physical_mutation.rs b/src/query/sql/src/executor/physical_plans/physical_mutation.rs index 31f949ee73b80..fa7ff06ea928d 100644 --- a/src/query/sql/src/executor/physical_plans/physical_mutation.rs +++ b/src/query/sql/src/executor/physical_plans/physical_mutation.rs @@ -81,8 +81,7 @@ pub type MatchExpr = Vec<(Option, Option, pub table_info: TableInfo, // (DataSchemaRef, Option, Vec,Vec) => (source_schema, condition, value_exprs) @@ -96,6 +95,7 @@ pub struct Mutation { pub table_meta_timestamps: TableMetaTimestamps, } +#[typetag::serde] impl IPhysicalPlan for Mutation { fn get_meta(&self) -> &PhysicalPlanMeta { &self.meta @@ -113,7 +113,7 @@ impl IPhysicalPlan for Mutation { Box::new(std::iter::once(&self.input)) } - fn children_mut<'a>(&'a self) -> Box> + 'a> { + fn children_mut<'a>(&'a mut self) -> Box> + 'a> { Box::new(std::iter::once(&mut self.input)) } } @@ -124,7 +124,7 @@ impl PhysicalPlanBuilder { s_expr: &SExpr, mutation: &crate::plans::Mutation, required: ColumnSet, - ) -> Result { + ) -> Result> { let crate::plans::Mutation { bind_context, metadata, @@ -165,7 +165,7 @@ impl PhysicalPlanBuilder { if *truncate_table { // Do truncate. - plan = PhysicalPlan::CommitSink(Box::new(CommitSink { + plan = Box::new(CommitSink { input: Box::new(plan), snapshot: mutation_build_info.table_snapshot, table_info: table_info.clone(), @@ -175,10 +175,10 @@ impl PhysicalPlanBuilder { }, update_stream_meta: vec![], deduplicated_label: unsafe { self.ctx.get_settings().get_deduplicate_label()? }, - plan_id: u32::MAX, recluster_info: None, + meta: PhysicalPlanMeta::new("CommitSink"), table_meta_timestamps: mutation_build_info.table_meta_timestamps, - })); + }); plan.adjust_plan_id(&mut 0); return Ok(plan); } @@ -232,9 +232,9 @@ impl PhysicalPlanBuilder { (None, None, MutationKind::Delete) }; - plan = PhysicalPlan::ColumnMutation(ColumnMutation { - plan_id: 0, - input: Box::new(plan), + plan = Box::new(ColumnMutation { + input: plan, + meta: PhysicalPlanMeta::new("ColumnMutation"), table_info: mutation_build_info.table_info.clone(), mutation_expr, computed_expr, @@ -246,18 +246,18 @@ impl PhysicalPlanBuilder { }); if *distributed { - plan = PhysicalPlan::Exchange(Exchange { - plan_id: 0, - input: Box::new(plan), + plan = Box::new(Exchange { + input: plan, kind: FragmentKind::Merge, keys: vec![], allow_adjust_parallelism: true, ignore_exchange: false, + meta: PhysicalPlanMeta::new("Exchange"), }); } - plan = PhysicalPlan::CommitSink(Box::new(CommitSink { - input: Box::new(plan), + plan = Box::new(CommitSink { + input: plan, snapshot: mutation_build_info.table_snapshot, table_info: table_info.clone(), // let's use update first, we will do some optimizations and select exact strategy @@ -267,10 +267,10 @@ impl PhysicalPlanBuilder { }, update_stream_meta: vec![], deduplicated_label: unsafe { self.ctx.get_settings().get_deduplicate_label()? }, - plan_id: u32::MAX, + meta: PhysicalPlanMeta::new("CommitSink"), recluster_info: None, table_meta_timestamps: mutation_build_info.table_meta_timestamps, - })); + }); plan.adjust_plan_id(&mut 0); return Ok(plan); @@ -287,23 +287,23 @@ impl PhysicalPlanBuilder { // different nodes update the same physical block simultaneously, data blocks that are needed // to insert just keep in local node. if *distributed && *row_id_shuffle && !is_not_matched_only { - plan = PhysicalPlan::Exchange(build_block_id_shuffle_exchange( + plan = build_block_id_shuffle_exchange( plan, bind_context, mutation_input_schema.clone(), database_name, &table_name, - )?); + )?; } // If the mutation type is FullOperation, we use row_id column to split a block // into matched and not matched parts. if matches!(strategy, MutationStrategy::MixedMatched) { - plan = PhysicalPlan::MutationSplit(Box::new(MutationSplit { - plan_id: 0, + plan = Box::new(MutationSplit { input: Box::new(plan), split_index: row_id_offset, - })); + meta: PhysicalPlanMeta::new("MutationSplit"), + }); } // Construct row fetch plan for lazy columns. @@ -313,7 +313,7 @@ impl PhysicalPlanBuilder { .get_table_lazy_columns(target_table_index) && !lazy_columns.is_empty() { - plan = PhysicalPlan::RowFetch(build_mutation_row_fetch( + plan = build_mutation_row_fetch( plan, metadata.clone(), mutation_input_schema.clone(), @@ -321,7 +321,7 @@ impl PhysicalPlanBuilder { lazy_columns.clone(), *target_table_index, row_id_offset, - )); + ); } let output_schema = plan.output_schema()?; @@ -432,8 +432,7 @@ impl PhysicalPlanBuilder { } } - plan = PhysicalPlan::MutationManipulate(Box::new(MutationManipulate { - plan_id: 0, + plan = Box::new(MutationManipulate { input: Box::new(plan.clone()), table_info: table_info.clone(), unmatched: unmatched.clone(), @@ -443,13 +442,14 @@ impl PhysicalPlanBuilder { row_id_idx: row_id_offset, can_try_update_column_only: *can_try_update_column_only, unmatched_schema: mutation_input_schema.clone(), - })); + meta: PhysicalPlanMeta::new("MutationManipulate"), + }); - plan = PhysicalPlan::MutationOrganize(Box::new(MutationOrganize { - plan_id: 0, + plan = Box::new(MutationOrganize { input: Box::new(plan.clone()), strategy: strategy.clone(), - })); + meta: PhysicalPlanMeta::new("MutationOrganize"), + }); let segments: Vec<_> = mutation_build_info .table_snapshot @@ -459,8 +459,8 @@ impl PhysicalPlanBuilder { .enumerate() .collect(); - let mutation = PhysicalPlan::Mutation(Box::new(Mutation { - input: Box::new(plan.clone()), + plan = Box::new(Mutation { + input: plan, table_info: table_info.clone(), unmatched, segments: segments.clone(), @@ -469,22 +469,20 @@ impl PhysicalPlanBuilder { target_table_index: *target_table_index, need_match: !is_not_matched_only, target_build_optimization: false, - plan_id: u32::MAX, + meta: PhysicalPlanMeta::new("Mutation"), table_meta_timestamps: mutation_build_info.table_meta_timestamps, - })); + }); - let commit_input = if !distributed { - mutation - } else { - PhysicalPlan::Exchange(Exchange { - plan_id: 0, - input: Box::new(mutation), + if distributed { + plan = Box::new(Exchange { + input: plan, kind: FragmentKind::Merge, keys: vec![], allow_adjust_parallelism: true, ignore_exchange: false, - }) - }; + meta: PhysicalPlanMeta::new("Exchange"), + }); + } let mutation_kind = match mutation_type { MutationType::Update | MutationType::Merge => MutationKind::Update, @@ -492,8 +490,8 @@ impl PhysicalPlanBuilder { }; // build mutation_aggregate - let mut physical_plan = PhysicalPlan::CommitSink(Box::new(CommitSink { - input: Box::new(commit_input), + let mut physical_plan = Box::new(CommitSink { + input: plan, snapshot: mutation_build_info.table_snapshot, table_info: table_info.clone(), // let's use update first, we will do some optimizations and select exact strategy @@ -503,10 +501,11 @@ impl PhysicalPlanBuilder { }, update_stream_meta: mutation_build_info.update_stream_meta, deduplicated_label: unsafe { self.ctx.get_settings().get_deduplicate_label()? }, - plan_id: u32::MAX, recluster_info: None, + meta: PhysicalPlanMeta::new("CommitSink"), table_meta_timestamps: mutation_build_info.table_meta_timestamps, - })); + }); + physical_plan.adjust_plan_id(&mut 0); Ok(physical_plan) } @@ -529,12 +528,12 @@ impl PhysicalPlanBuilder { } pub fn build_block_id_shuffle_exchange( - plan: PhysicalPlan, + plan: Box, bind_context: &BindContext, mutation_input_schema: Arc, database_name: &str, table_name: &str, -) -> Result { +) -> Result> { let mut row_id_column = None; for column_binding in bind_context.columns.iter() { if BindContext::match_column_binding( @@ -584,25 +583,25 @@ pub fn build_block_id_shuffle_exchange( &BUILTIN_FUNCTIONS, )?; - Ok(Exchange { - plan_id: 0, - input: Box::new(plan), + Ok(Box::new(Exchange { + input: plan, kind: FragmentKind::Normal, + meta: PhysicalPlanMeta::new("Exchange"), keys: vec![block_id_shuffle_key.as_remote_expr()], allow_adjust_parallelism: true, ignore_exchange: false, - }) + })) } fn build_mutation_row_fetch( - plan: PhysicalPlan, + plan: Box, metadata: MetadataRef, mutation_input_schema: Arc, strategy: MutationStrategy, lazy_columns: ColumnSet, target_table_index: usize, row_id_offset: usize, -) -> RowFetch { +) -> Box { let metadata = metadata.read(); let lazy_columns = lazy_columns @@ -644,16 +643,16 @@ fn build_mutation_row_fetch( false, ); - RowFetch { - plan_id: 0, - input: Box::new(plan), + Box::new(RowFetch { + input: plan, source: Box::new(source), row_id_col_offset: row_id_offset, cols_to_fetch, fetched_fields, need_wrap_nullable, stat_info: None, - } + meta: PhysicalPlanMeta::new("RowFetch"), + }) } #[allow(clippy::too_many_arguments)] @@ -673,7 +672,7 @@ pub fn generate_update_list( Box::new(DataType::Boolean), Visibility::Visible, ) - .build(); + .build(); let predicate = ScalarExpr::BoundColumnRef(BoundColumnRef { span: None, column }); update_list.iter().try_fold( @@ -758,7 +757,7 @@ pub fn mutation_update_expr( Box::new(DataType::Boolean), Visibility::Visible, ) - .build(); + .build(); ScalarExpr::BoundColumnRef(BoundColumnRef { span: None, column }) } else { ScalarExpr::ConstantExpr(ConstantExpr { diff --git a/src/query/sql/src/executor/physical_plans/physical_mutation_into_organize.rs b/src/query/sql/src/executor/physical_plans/physical_mutation_into_organize.rs index de83e71a75677..2371b56c634ba 100644 --- a/src/query/sql/src/executor/physical_plans/physical_mutation_into_organize.rs +++ b/src/query/sql/src/executor/physical_plans/physical_mutation_into_organize.rs @@ -21,12 +21,12 @@ use crate::executor::physical_plan::PhysicalPlan; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct MutationOrganize { - pub plan_id: u32, - meta: PhysicalPlanMeta, + pub meta: PhysicalPlanMeta, pub input: Box, pub strategy: MutationStrategy, } +#[typetag::serde] impl IPhysicalPlan for MutationOrganize { fn get_meta(&self) -> &PhysicalPlanMeta { &self.meta @@ -40,7 +40,7 @@ impl IPhysicalPlan for MutationOrganize { Box::new(std::iter::once(&self.input)) } - fn children_mut<'a>(&'a self) -> Box> + 'a> { + fn children_mut<'a>(&'a mut self) -> Box> + 'a> { Box::new(std::iter::once(&mut self.input)) } } diff --git a/src/query/sql/src/executor/physical_plans/physical_mutation_into_split.rs b/src/query/sql/src/executor/physical_plans/physical_mutation_into_split.rs index ebb0704386aa8..dbc836bbf31d4 100644 --- a/src/query/sql/src/executor/physical_plans/physical_mutation_into_split.rs +++ b/src/query/sql/src/executor/physical_plans/physical_mutation_into_split.rs @@ -20,12 +20,12 @@ use crate::IndexType; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct MutationSplit { - pub plan_id: u32, - meta: PhysicalPlanMeta, + pub meta: PhysicalPlanMeta, pub input: Box, pub split_index: IndexType, } +#[typetag::serde] impl IPhysicalPlan for MutationSplit { fn get_meta(&self) -> &PhysicalPlanMeta { &self.meta @@ -39,7 +39,7 @@ impl IPhysicalPlan for MutationSplit { Box::new(std::iter::once(&self.input)) } - fn children_mut<'a>(&'a self) -> Box> + 'a> { + fn children_mut<'a>(&'a mut self) -> Box> + 'a> { Box::new(std::iter::once(&mut self.input)) } } diff --git a/src/query/sql/src/executor/physical_plans/physical_mutation_manipulate.rs b/src/query/sql/src/executor/physical_plans/physical_mutation_manipulate.rs index cf3194df9781d..23b8fe56d0a3f 100644 --- a/src/query/sql/src/executor/physical_plans/physical_mutation_manipulate.rs +++ b/src/query/sql/src/executor/physical_plans/physical_mutation_manipulate.rs @@ -28,8 +28,7 @@ pub type MatchExpr = Vec<(Option, Option, pub table_info: TableInfo, // (DataSchemaRef, Option, Vec,Vec) => (source_schema, condition, value_exprs) @@ -45,6 +44,7 @@ pub struct MutationManipulate { pub unmatched_schema: DataSchemaRef, } +#[typetag::serde] impl IPhysicalPlan for MutationManipulate { fn get_meta(&self) -> &PhysicalPlanMeta { &self.meta @@ -58,7 +58,7 @@ impl IPhysicalPlan for MutationManipulate { Box::new(std::iter::once(&self.input)) } - fn children_mut<'a>(&'a self) -> Box> + 'a> { + fn children_mut<'a>(&'a mut self) -> Box> + 'a> { Box::new(std::iter::once(&mut self.input)) } } diff --git a/src/query/sql/src/executor/physical_plans/physical_mutation_source.rs b/src/query/sql/src/executor/physical_plans/physical_mutation_source.rs index c0180e7bfa567..f41513a60d5de 100644 --- a/src/query/sql/src/executor/physical_plans/physical_mutation_source.rs +++ b/src/query/sql/src/executor/physical_plans/physical_mutation_source.rs @@ -36,9 +36,7 @@ use crate::ScalarExpr; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct MutationSource { - // A unique id of operator in a `PhysicalPlan` tree, only used for display. - pub plan_id: u32, - meta: PhysicalPlanMeta, + pub meta: PhysicalPlanMeta, pub table_index: IndexType, pub table_info: TableInfo, pub filters: Option, @@ -51,6 +49,7 @@ pub struct MutationSource { pub statistics: PartStatistics, } +#[typetag::serde] impl IPhysicalPlan for MutationSource { fn get_meta(&self) -> &PhysicalPlanMeta { &self.meta @@ -73,7 +72,7 @@ impl PhysicalPlanBuilder { pub(crate) async fn build_mutation_source( &mut self, mutation_source: &crate::plans::MutationSource, - ) -> Result { + ) -> Result> { let filters = if !mutation_source.predicates.is_empty() { Some(create_push_down_filters( &self.ctx.get_function_context()?, @@ -114,8 +113,7 @@ impl PhysicalPlanBuilder { let truncate_table = mutation_source.mutation_type == MutationType::Delete && filters.is_none(); - Ok(PhysicalPlan::MutationSource(MutationSource { - plan_id: 0, + Ok(Box::new(MutationSource { table_index: mutation_source.table_index, output_schema, table_info: mutation_info.table_info.clone(), @@ -123,6 +121,7 @@ impl PhysicalPlanBuilder { input_type: mutation_source.mutation_type.clone(), read_partition_columns: mutation_source.read_partition_columns.clone(), truncate_table, + meta: PhysicalPlanMeta::new("MutationSource"), partitions: mutation_info.partitions.clone(), statistics: mutation_info.statistics.clone(), })) diff --git a/src/query/sql/src/executor/physical_plans/physical_project_set.rs b/src/query/sql/src/executor/physical_plans/physical_project_set.rs index 630a3a80b33a4..d58f5c7d252f8 100644 --- a/src/query/sql/src/executor/physical_plans/physical_project_set.rs +++ b/src/query/sql/src/executor/physical_plans/physical_project_set.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use itertools::Itertools; use databend_common_catalog::plan::DataSourcePlan; use databend_common_exception::Result; use databend_common_expression::ConstantFolder; @@ -31,8 +32,6 @@ use crate::TypeCheck; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct ProjectSet { - // A unique id of operator in a `PhysicalPlan` tree, only used for display. - pub plan_id: u32, meta: PhysicalPlanMeta, pub projections: ColumnSet, pub input: Box, @@ -42,6 +41,7 @@ pub struct ProjectSet { pub stat_info: Option, } +#[typetag::serde] impl IPhysicalPlan for ProjectSet { fn get_meta(&self) -> &PhysicalPlanMeta { &self.meta @@ -72,7 +72,7 @@ impl IPhysicalPlan for ProjectSet { Box::new(std::iter::once(&self.input)) } - fn children_mut<'a>(&'a self) -> Box> + 'a> { + fn children_mut<'a>(&'a mut self) -> Box> + 'a> { Box::new(std::iter::once(&mut self.input)) } @@ -80,6 +80,14 @@ impl IPhysicalPlan for ProjectSet { fn try_find_single_data_source(&self) -> Option<&DataSourcePlan> { self.input.try_find_single_data_source() } + + fn get_desc(&self) -> Result { + Ok(self + .srf_exprs + .iter() + .map(|(x, _)| x.as_expr(&BUILTIN_FUNCTIONS).sql_display()) + .join(", ")) + } } impl PhysicalPlanBuilder { @@ -89,7 +97,7 @@ impl PhysicalPlanBuilder { project_set: &crate::plans::ProjectSet, mut required: ColumnSet, stat_info: PlanStatsInfo, - ) -> Result { + ) -> Result> { // 1. Prune unused Columns. let column_projections = required.clone().into_iter().collect::>(); for s in project_set.srfs.iter() { @@ -119,8 +127,8 @@ impl PhysicalPlanBuilder { } } - Ok(PhysicalPlan::ProjectSet(ProjectSet { - plan_id: 0, + Ok(Box::new(ProjectSet { + meta: PhysicalPlanMeta::new("Unnest"), input: Box::new(input), srf_exprs, projections, diff --git a/src/query/sql/src/executor/physical_plans/physical_r_cte_scan.rs b/src/query/sql/src/executor/physical_plans/physical_r_cte_scan.rs index 929946404a18a..b544c37402333 100644 --- a/src/query/sql/src/executor/physical_plans/physical_r_cte_scan.rs +++ b/src/query/sql/src/executor/physical_plans/physical_r_cte_scan.rs @@ -24,15 +24,14 @@ use crate::executor::PhysicalPlanBuilder; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct RecursiveCteScan { - // A unique id of operator in a `PhysicalPlan` tree, only used for display. - pub plan_id: u32, meta: PhysicalPlanMeta, pub output_schema: DataSchemaRef, pub table_name: String, pub stat: PlanStatsInfo, } -impl IPhysicalPlan for RecursiveCteScan { +#[typetag::serde] +impl IPhysicalPlan for RecursiveCteScan { fn get_meta(&self) -> &PhysicalPlanMeta { &self.meta } @@ -51,9 +50,9 @@ impl PhysicalPlanBuilder { &mut self, recursive_cte_scan: &crate::plans::RecursiveCteScan, stat_info: PlanStatsInfo, - ) -> Result { - Ok(PhysicalPlan::RecursiveCteScan(RecursiveCteScan { - plan_id: 0, + ) -> Result> { + Ok(Box::new(RecursiveCteScan { + meta: PhysicalPlanMeta::new("RecursiveCteScan"), output_schema: DataSchemaRefExt::create(recursive_cte_scan.fields.clone()), table_name: recursive_cte_scan.table_name.clone(), stat: stat_info, diff --git a/src/query/sql/src/executor/physical_plans/physical_range_join.rs b/src/query/sql/src/executor/physical_plans/physical_range_join.rs index 1b1bd4a5b9128..95b3ef02d5b78 100644 --- a/src/query/sql/src/executor/physical_plans/physical_range_join.rs +++ b/src/query/sql/src/executor/physical_plans/physical_range_join.rs @@ -39,11 +39,9 @@ use crate::TypeCheck; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct RangeJoin { - // A unique id of operator in a `PhysicalPlan` tree, only used for display. - pub plan_id: u32, - meta: PhysicalPlanMeta, - pub left: Box, - pub right: Box, + pub meta: PhysicalPlanMeta, + pub left: Box, + pub right: Box, // The first two conditions: (>, >=, <, <=) // Condition's left/right side only contains one table's column pub conditions: Vec, @@ -58,6 +56,7 @@ pub struct RangeJoin { pub stat_info: Option, } +#[typetag::serde] impl IPhysicalPlan for RangeJoin { fn get_meta(&self) -> &PhysicalPlanMeta { &self.meta @@ -75,9 +74,35 @@ impl IPhysicalPlan for RangeJoin { Box::new(std::iter::once(&self.left).chain(std::iter::once(&self.right))) } - fn children_mut<'a>(&'a self) -> Box> + 'a> { + fn children_mut<'a>(&'a mut self) -> Box> + 'a> { Box::new(std::iter::once(&mut self.left).chain(std::iter::once(&mut self.right))) } + + fn get_desc(&self) -> Result { + let mut condition = self + .conditions + .iter() + .map(|condition| { + let left = condition + .left_expr + .as_expr(&BUILTIN_FUNCTIONS) + .sql_display(); + let right = condition + .right_expr + .as_expr(&BUILTIN_FUNCTIONS) + .sql_display(); + format!("{left} {:?} {right}", condition.operator) + }) + .collect::>(); + + condition.extend( + self.other_conditions + .iter() + .map(|x| x.as_expr(&BUILTIN_FUNCTIONS).sql_display()), + ); + + Ok(condition.join(" AND ")) + } } #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] @@ -103,7 +128,7 @@ impl PhysicalPlanBuilder { right_required: ColumnSet, mut range_conditions: Vec, mut other_conditions: Vec, - ) -> Result { + ) -> Result> { let left_prop = RelExpr::with_s_expr(s_expr.child(1)?).derive_relational_prop()?; let right_prop = RelExpr::with_s_expr(s_expr.child(0)?).derive_relational_prop()?; @@ -139,10 +164,10 @@ impl PhysicalPlanBuilder { .collect::>(), ); - Ok(PhysicalPlan::RangeJoin(RangeJoin { - plan_id: 0, + Ok(Box::new(RangeJoin { left: left_side, right: right_side, + meta: PhysicalPlanMeta::new("RangeJoin"), conditions: range_conditions .iter() .map(|scalar| { diff --git a/src/query/sql/src/executor/physical_plans/physical_recluster.rs b/src/query/sql/src/executor/physical_plans/physical_recluster.rs index b510b4f295cbd..15caa374197ff 100644 --- a/src/query/sql/src/executor/physical_plans/physical_recluster.rs +++ b/src/query/sql/src/executor/physical_plans/physical_recluster.rs @@ -22,13 +22,13 @@ use databend_common_exception::Result; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct Recluster { - pub plan_id: u32, - meta: PhysicalPlanMeta, + pub meta: PhysicalPlanMeta, pub tasks: Vec, pub table_info: TableInfo, pub table_meta_timestamps: TableMetaTimestamps, } +#[typetag::serde] impl IPhysicalPlan for Recluster { fn get_meta(&self) -> &PhysicalPlanMeta { &self.meta @@ -41,8 +41,7 @@ impl IPhysicalPlan for Recluster { #[derive(serde::Serialize, serde::Deserialize, Clone, Debug)] pub struct HilbertPartition { - pub plan_id: u32, - meta: PhysicalPlanMeta, + pub meta: PhysicalPlanMeta, pub input: Box, pub table_info: TableInfo, pub num_partitions: usize, @@ -50,6 +49,7 @@ pub struct HilbertPartition { pub rows_per_block: usize, } +#[typetag::serde] impl IPhysicalPlan for HilbertPartition { fn get_meta(&self) -> &PhysicalPlanMeta { &self.meta @@ -67,7 +67,7 @@ impl IPhysicalPlan for HilbertPartition { Box::new(std::iter::once(&self.input)) } - fn children_mut<'a>(&'a self) -> Box> + 'a> { + fn children_mut<'a>(&'a mut self) -> Box> + 'a> { Box::new(std::iter::once(&mut self.input)) } } diff --git a/src/query/sql/src/executor/physical_plans/physical_replace_async_source.rs b/src/query/sql/src/executor/physical_plans/physical_replace_async_source.rs index 570a0d92ce668..89c6f9b37d7a9 100644 --- a/src/query/sql/src/executor/physical_plans/physical_replace_async_source.rs +++ b/src/query/sql/src/executor/physical_plans/physical_replace_async_source.rs @@ -19,12 +19,12 @@ use crate::plans::InsertValue; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct ReplaceAsyncSourcer { - pub plan_id: u32, - meta: PhysicalPlanMeta, + pub meta: PhysicalPlanMeta, pub schema: DataSchemaRef, pub source: InsertValue, } +#[typetag::serde] impl IPhysicalPlan for ReplaceAsyncSourcer { fn get_meta(&self) -> &PhysicalPlanMeta { &self.meta diff --git a/src/query/sql/src/executor/physical_plans/physical_replace_deduplicate.rs b/src/query/sql/src/executor/physical_plans/physical_replace_deduplicate.rs index 7d91fff666cd1..2435988226d3e 100644 --- a/src/query/sql/src/executor/physical_plans/physical_replace_deduplicate.rs +++ b/src/query/sql/src/executor/physical_plans/physical_replace_deduplicate.rs @@ -29,8 +29,7 @@ use databend_common_exception::Result; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct ReplaceDeduplicate { - pub plan_id: u32, - meta: PhysicalPlanMeta, + pub meta: PhysicalPlanMeta, pub input: Box, pub on_conflicts: Vec, pub bloom_filter_column_indexes: Vec, @@ -43,6 +42,7 @@ pub struct ReplaceDeduplicate { pub delete_when: Option<(RemoteExpr, String)>, } +#[typetag::serde] impl IPhysicalPlan for ReplaceDeduplicate { fn get_meta(&self) -> &PhysicalPlanMeta { &self.meta @@ -60,7 +60,7 @@ impl IPhysicalPlan for ReplaceDeduplicate { Box::new(std::iter::once(&self.input)) } - fn children_mut<'a>(&'a self) -> Box> + 'a> { + fn children_mut<'a>(&'a mut self) -> Box> + 'a> { Box::new(std::iter::once(&mut self.input)) } } diff --git a/src/query/sql/src/executor/physical_plans/physical_replace_into.rs b/src/query/sql/src/executor/physical_plans/physical_replace_into.rs index bd2815dfbb745..079ff5ac5859f 100644 --- a/src/query/sql/src/executor/physical_plans/physical_replace_into.rs +++ b/src/query/sql/src/executor/physical_plans/physical_replace_into.rs @@ -26,10 +26,7 @@ use databend_common_exception::Result; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct ReplaceInto { - /// A unique id of operator in a `PhysicalPlan` tree. - pub plan_id: u32, - - meta: PhysicalPlanMeta, + pub meta: PhysicalPlanMeta, pub input: Box, pub block_thresholds: BlockThresholds, pub table_info: TableInfo, @@ -41,6 +38,7 @@ pub struct ReplaceInto { pub table_meta_timestamps: TableMetaTimestamps, } +#[typetag::serde] impl IPhysicalPlan for ReplaceInto { fn get_meta(&self) -> &PhysicalPlanMeta { &self.meta @@ -58,7 +56,7 @@ impl IPhysicalPlan for ReplaceInto { Box::new(std::iter::once(&self.input)) } - fn children_mut<'a>(&'a self) -> Box> + 'a> { + fn children_mut<'a>(&'a mut self) -> Box> + 'a> { Box::new(std::iter::once(&mut self.input)) } } diff --git a/src/query/sql/src/executor/physical_plans/physical_row_fetch.rs b/src/query/sql/src/executor/physical_plans/physical_row_fetch.rs index 1c0bab38653f2..003f4c87bce3a 100644 --- a/src/query/sql/src/executor/physical_plans/physical_row_fetch.rs +++ b/src/query/sql/src/executor/physical_plans/physical_row_fetch.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use itertools::Itertools; use databend_common_catalog::plan::DataSourcePlan; use databend_common_catalog::plan::Projection; use databend_common_exception::Result; @@ -24,9 +25,7 @@ use crate::executor::{IPhysicalPlan, PhysicalPlan, PhysicalPlanMeta}; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct RowFetch { - // A unique id of operator in a `PhysicalPlan` tree, only used for display. - pub plan_id: u32, - meta: PhysicalPlanMeta, + pub meta: PhysicalPlanMeta, pub input: Box, // cloned from `input`. pub source: Box, @@ -40,6 +39,7 @@ pub struct RowFetch { pub stat_info: Option, } +#[typetag::serde] impl IPhysicalPlan for RowFetch { fn get_meta(&self) -> &PhysicalPlanMeta { &self.meta @@ -59,7 +59,7 @@ impl IPhysicalPlan for RowFetch { Box::new(std::iter::once(&self.input)) } - fn children_mut<'a>(&'a self) -> Box> + 'a> { + fn children_mut<'a>(&'a mut self) -> Box> + 'a> { Box::new(std::iter::once(&mut self.input)) } @@ -67,4 +67,10 @@ impl IPhysicalPlan for RowFetch { fn try_find_single_data_source(&self) -> Option<&DataSourcePlan> { self.input.try_find_single_data_source() } + + fn get_desc(&self) -> Result { + let table_schema = self.source.source_info.schema(); + let projected_schema = self.cols_to_fetch.project_schema(&table_schema); + Ok(projected_schema.fields.iter().map(|f| f.name()).join(", ")) + } } diff --git a/src/query/sql/src/executor/physical_plans/physical_sort.rs b/src/query/sql/src/executor/physical_plans/physical_sort.rs index 1ba82321ed8fd..72f85be8846ef 100644 --- a/src/query/sql/src/executor/physical_plans/physical_sort.rs +++ b/src/query/sql/src/executor/physical_plans/physical_sort.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use itertools::Itertools; use databend_common_catalog::plan::DataSourcePlan; use databend_common_exception::Result; use databend_common_expression::types::DataType; @@ -35,9 +36,7 @@ use crate::IndexType; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct Sort { - /// A unique id of operator in a `PhysicalPlan` tree, only used for display. - pub plan_id: u32, - meta: PhysicalPlanMeta, + pub meta: PhysicalPlanMeta, pub input: Box, pub order_by: Vec, /// limit = Limit.limit + Limit.offset @@ -51,6 +50,7 @@ pub struct Sort { pub stat_info: Option, } +#[typetag::serde] impl IPhysicalPlan for Sort { fn get_meta(&self) -> &PhysicalPlanMeta { &self.meta @@ -102,7 +102,7 @@ impl IPhysicalPlan for Sort { Box::new(std::iter::once(&self.input)) } - fn children_mut<'a>(&'a self) -> Box> + 'a> { + fn children_mut<'a>(&'a mut self) -> Box> + 'a> { Box::new(std::iter::once(&mut self.input)) } @@ -110,6 +110,22 @@ impl IPhysicalPlan for Sort { fn try_find_single_data_source(&self) -> Option<&DataSourcePlan> { self.input.try_find_single_data_source() } + + fn get_desc(&self) -> Result { + Ok(self + .order_by + .iter() + .map(|x| { + format!( + "{}{}{}", + x.display_name, + if x.asc { "" } else { " DESC" }, + if x.nulls_first { " NULLS FIRST" } else { "" }, + ) + }) + .join(", ") + ) + } } impl Sort { @@ -134,7 +150,7 @@ impl PhysicalPlanBuilder { sort: &crate::plans::Sort, mut required: ColumnSet, stat_info: PlanStatsInfo, - ) -> Result { + ) -> Result { // 1. Prune unused Columns. sort.items.iter().for_each(|s| { required.insert(s.index); @@ -167,8 +183,8 @@ impl PhysicalPlanBuilder { .map(|v| v.index) .collect::>(); - return Ok(PhysicalPlan::WindowPartition(WindowPartition { - plan_id: 0, + return Ok(Box::new(WindowPartition { + meta: PhysicalPlanMeta::new("WindowPartition"), input: Box::new(input_plan.clone()), partition_by: window_partition.clone(), order_by: order_by.clone(), @@ -187,14 +203,14 @@ impl PhysicalPlanBuilder { }; // 2. Build physical plan. - Ok(PhysicalPlan::Sort(Sort { - plan_id: 0, - input: Box::new(input_plan), + Ok(Box::new(Sort { order_by, + pre_projection, + input: Box::new(input_plan), limit: sort.limit, after_exchange: sort.after_exchange, - pre_projection, stat_info: Some(stat_info), + meta: PhysicalPlanMeta::new("Sort"), })) } } diff --git a/src/query/sql/src/executor/physical_plans/physical_table_scan.rs b/src/query/sql/src/executor/physical_plans/physical_table_scan.rs index 4b59931acc661..eb19aa9c2a268 100644 --- a/src/query/sql/src/executor/physical_plans/physical_table_scan.rs +++ b/src/query/sql/src/executor/physical_plans/physical_table_scan.rs @@ -20,7 +20,7 @@ use std::sync::Arc; use databend_common_ast::parser::token::TokenKind; use databend_common_ast::parser::tokenize_sql; use databend_common_catalog::catalog::CatalogManager; -use databend_common_catalog::plan::{DataSourcePlan, PartStatistics, PartitionsShuffleKind}; +use databend_common_catalog::plan::{DataSourceInfo, DataSourcePlan, PartStatistics, PartitionsShuffleKind}; use databend_common_catalog::plan::Filters; use databend_common_catalog::plan::InternalColumn; use databend_common_catalog::plan::PrewhereInfo; @@ -74,8 +74,6 @@ use crate::DUMMY_TABLE_INDEX; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct TableScan { meta: PhysicalPlanMeta, - // A unique id of operator in a `PhysicalPlan` tree, only used for display. - pub plan_id: u32, pub scan_id: usize, pub name_mapping: BTreeMap, pub source: Box, @@ -85,6 +83,7 @@ pub struct TableScan { pub stat_info: Option, } +#[typetag::serde] impl IPhysicalPlan for TableScan { fn get_meta(&self) -> &PhysicalPlanMeta { &self.meta @@ -133,9 +132,64 @@ impl IPhysicalPlan for TableScan { fn is_warehouse_distributed_plan(&self) -> bool { self.source.parts.kind == PartitionsShuffleKind::BroadcastWarehouse } + + fn get_desc(&self) -> Result { + Ok(format!( + "{}.{}", + self.source.source_info.catalog_name(), + self.source.source_info.desc() + )) + } + + fn get_labels(&self) -> Result>> { + Ok(HashMap::from([ + (String::from("Full table name"), vec![format!( + "{}.{}", + self.source.source_info.catalog_name(), + self.source.source_info.desc() + )]), + ( + format!( + "Columns ({} / {})", + self.output_schema()?.num_fields(), + std::cmp::max( + self.output_schema()?.num_fields(), + self.source.source_info.schema().num_fields(), + ) + ), + self.name_mapping.keys().cloned().collect(), + ), + (String::from("Total partitions"), vec![self + .source + .statistics + .partitions_total + .to_string()]) + ])) + } } impl TableScan { + pub fn new(scan_id: usize, name_mapping: BTreeMap, source: Box, table_index: Option, stat_info: Option, internal_column: Option>) -> Box { + let name = match &source.source_info { + DataSourceInfo::TableSource(_) => "TableScan".to_string(), + DataSourceInfo::StageSource(_) => "StageScan".to_string(), + DataSourceInfo::ParquetSource(_) => "ParquetScan".to_string(), + DataSourceInfo::ResultScanSource(_) => "ResultScan".to_string(), + DataSourceInfo::ORCSource(_) => "OrcScan".to_string(), + }; + + Box::new(TableScan { + meta: PhysicalPlanMeta::new(name), + source, + scan_id, + name_mapping, + table_index, + stat_info, + internal_column, + }) + } + + pub fn output_fields( schema: TableSchemaRef, name_mapping: &BTreeMap, @@ -166,7 +220,7 @@ impl PhysicalPlanBuilder { scan: &crate::plans::Scan, required: ColumnSet, stat_info: PlanStatsInfo, - ) -> Result { + ) -> Result> { // 1. Prune unused Columns. // Some table may not have any column, // e.g. `system.sync_crash_me` @@ -354,30 +408,29 @@ impl PhysicalPlanBuilder { metadata.set_table_source(scan.table_index, source.clone()); } - let mut plan = PhysicalPlan::TableScan(TableScan { - plan_id: 0, - scan_id: scan.scan_id, + let mut plan = TableScan::new( + scan.scan_id, name_mapping, - source: Box::new(source), - table_index: Some(scan.table_index), - stat_info: Some(stat_info), + Box::new(source), + Some(scan.table_index), + Some(stat_info), internal_column, - }); + ); // Update stream columns if needed. if scan.update_stream_columns { - plan = PhysicalPlan::AddStreamColumn(Box::new(AddStreamColumn::new( + plan = AddStreamColumn::new( &self.metadata, plan, scan.table_index, table.get_table_info().ident.seq, - )?)); + )?; } Ok(plan) } - pub(crate) async fn build_dummy_table_scan(&mut self) -> Result { + pub(crate) async fn build_dummy_table_scan(&mut self) -> Result> { let catalogs = CatalogManager::instance(); let table = catalogs .get_default_catalog(self.ctx.session_state())? @@ -391,17 +444,17 @@ impl PhysicalPlanBuilder { let source = table .read_plan(self.ctx.clone(), None, None, false, self.dry_run) .await?; - Ok(PhysicalPlan::TableScan(TableScan { - plan_id: 0, - scan_id: DUMMY_TABLE_INDEX, - name_mapping: BTreeMap::from([("dummy".to_string(), DUMMY_COLUMN_INDEX)]), - source: Box::new(source), - table_index: Some(DUMMY_TABLE_INDEX), - stat_info: Some(PlanStatsInfo { + + Ok(TableScan::new( + DUMMY_TABLE_INDEX, + BTreeMap::from([("dummy".to_string(), DUMMY_COLUMN_INDEX)]), + Box::new(source), + Some(DUMMY_TABLE_INDEX), + Some(PlanStatsInfo { estimated_rows: 1.0, }), - internal_column: None, - })) + None, + )) } fn push_downs( diff --git a/src/query/sql/src/executor/physical_plans/physical_udf.rs b/src/query/sql/src/executor/physical_plans/physical_udf.rs index f8bf1534a05d2..f51443021aa09 100644 --- a/src/query/sql/src/executor/physical_plans/physical_udf.rs +++ b/src/query/sql/src/executor/physical_plans/physical_udf.rs @@ -13,6 +13,7 @@ // limitations under the License. use std::collections::BTreeMap; +use itertools::Itertools; use databend_common_catalog::plan::DataSourcePlan; use databend_common_exception::ErrorCode; use databend_common_exception::Result; @@ -33,9 +34,7 @@ use crate::ScalarExpr; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct Udf { - // A unique id of operator in a `PhysicalPlan` tree, only used for display. - pub plan_id: u32, - meta: PhysicalPlanMeta, + pub meta: PhysicalPlanMeta, pub input: Box, pub udf_funcs: Vec, pub script_udf: bool, @@ -43,6 +42,7 @@ pub struct Udf { pub stat_info: Option, } +#[typetag::serde] impl IPhysicalPlan for Udf { fn get_meta(&self) -> &PhysicalPlanMeta { &self.meta @@ -67,7 +67,7 @@ impl IPhysicalPlan for Udf { Box::new(std::iter::once(&self.input)) } - fn children_mut<'a>(&'a self) -> Box> + 'a> { + fn children_mut<'a>(&'a mut self) -> Box> + 'a> { Box::new(std::iter::once(&mut self.input)) } @@ -75,6 +75,14 @@ impl IPhysicalPlan for Udf { fn try_find_single_data_source(&self) -> Option<&DataSourcePlan> { self.input.try_find_single_data_source() } + + fn get_desc(&self) -> Result { + Ok(self + .udf_funcs + .iter() + .map(|x| format!("{}({})", x.func_name, x.arg_exprs.join(", "))) + .join(", ")) + } } #[derive(Clone, Debug, Eq, Hash, PartialEq, serde::Serialize, serde::Deserialize)] @@ -97,7 +105,7 @@ impl PhysicalPlanBuilder { udf_plan: &crate::plans::Udf, mut required: ColumnSet, stat_info: PlanStatsInfo, - ) -> Result { + ) -> Result> { // 1. Prune unused Columns. let mut used = vec![]; for item in udf_plan.items.iter() { @@ -168,12 +176,12 @@ impl PhysicalPlanBuilder { }) .collect::>>()?; - Ok(PhysicalPlan::Udf(Udf { - plan_id: 0, + Ok(Box::new(Udf { input: Box::new(input), udf_funcs, script_udf: udf_plan.script_udf, stat_info: Some(stat_info), + meta: PhysicalPlanMeta::new("Udf"), })) } } diff --git a/src/query/sql/src/executor/physical_plans/physical_union_all.rs b/src/query/sql/src/executor/physical_plans/physical_union_all.rs index 12e8b2c558334..29ff2198645f9 100644 --- a/src/query/sql/src/executor/physical_plans/physical_union_all.rs +++ b/src/query/sql/src/executor/physical_plans/physical_union_all.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use itertools::Itertools; use databend_common_exception::Result; use databend_common_expression::DataField; use databend_common_expression::DataSchema; @@ -30,11 +31,9 @@ use crate::TypeCheck; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct UnionAll { - // A unique id of operator in a `PhysicalPlan` tree, only used for display. - pub plan_id: u32, meta: PhysicalPlanMeta, - pub left: Box, - pub right: Box, + pub left: Box, + pub right: Box, pub left_outputs: Vec<(IndexType, Option)>, pub right_outputs: Vec<(IndexType, Option)>, pub schema: DataSchemaRef, @@ -44,6 +43,7 @@ pub struct UnionAll { pub stat_info: Option, } +#[typetag::serde] impl IPhysicalPlan for UnionAll { fn get_meta(&self) -> &PhysicalPlanMeta { &self.meta @@ -61,9 +61,18 @@ impl IPhysicalPlan for UnionAll { Box::new(std::iter::once(&self.left).chain(std::iter::once(&self.right))) } - fn children_mut<'a>(&'a self) -> Box> + 'a> { + fn children_mut<'a>(&'a mut self) -> Box> + 'a> { Box::new(std::iter::once(&mut self.left).chain(std::iter::once(&mut self.right))) } + + fn get_desc(&self) -> Result { + Ok(self + .left_outputs + .iter() + .zip(self.right_outputs.iter()) + .map(|(l, r)| format!("#{} <- #{}", l.0, r.0)) + .join(", ")) + } } impl PhysicalPlanBuilder { @@ -73,7 +82,7 @@ impl PhysicalPlanBuilder { union_all: &crate::plans::UnionAll, mut required: ColumnSet, stat_info: PlanStatsInfo, - ) -> Result { + ) -> Result> { // 1. Prune unused Columns. let metadata = self.metadata.read().clone(); let lazy_columns = metadata.lazy_columns(); @@ -145,14 +154,13 @@ impl PhysicalPlanBuilder { let right_outputs = process_outputs(&union_all.right_outputs, &right_required, &right_schema)?; - Ok(PhysicalPlan::UnionAll(UnionAll { - plan_id: 0, - left: Box::new(left_plan), - right: Box::new(right_plan), + Ok(Box::new(UnionAll { + left: left_plan, + right: right_plan, left_outputs, right_outputs, schema: DataSchemaRefExt::create(fields), - + meta: PhysicalPlanMeta::new("UnionAll"), cte_scan_names: union_all.cte_scan_names.clone(), stat_info: Some(stat_info), })) diff --git a/src/query/sql/src/executor/physical_plans/physical_window.rs b/src/query/sql/src/executor/physical_plans/physical_window.rs index 99ef9dcf6c2f4..ded448a1b34c7 100644 --- a/src/query/sql/src/executor/physical_plans/physical_window.rs +++ b/src/query/sql/src/executor/physical_plans/physical_window.rs @@ -48,9 +48,7 @@ use crate::TypeCheck; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct Window { - // A unique id of operator in a `PhysicalPlan` tree, only used for display. - pub plan_id: u32, - meta: PhysicalPlanMeta, + pub meta: PhysicalPlanMeta, pub index: IndexType, pub input: Box, pub func: WindowFunction, @@ -60,6 +58,7 @@ pub struct Window { pub limit: Option, } +#[typetag::serde] impl IPhysicalPlan for Window { fn get_meta(&self) -> &PhysicalPlanMeta { &self.meta @@ -84,7 +83,7 @@ impl IPhysicalPlan for Window { Box::new(std::iter::once(&self.input)) } - fn children_mut<'a>(&'a self) -> Box> + 'a> { + fn children_mut<'a>(&'a mut self) -> Box> + 'a> { Box::new(std::iter::once(&mut self.input)) } @@ -92,6 +91,31 @@ impl IPhysicalPlan for Window { fn try_find_single_data_source(&self) -> Option<&DataSourcePlan> { self.input.try_find_single_data_source() } + + fn get_desc(&self) -> Result { + let partition_by = self + .partition_by + .iter() + .map(|x| x.to_string()) + .collect::>() + .join(", "); + + let order_by = self + .order_by + .iter() + .map(|x| { + format!( + "{}{}{}", + x.display_name, + if x.asc { "" } else { " DESC" }, + if x.nulls_first { " NULLS FIRST" } else { "" }, + ) + }) + .collect::>() + .join(", "); + + Ok(format!("partition by {}, order by {}", partition_by, order_by)) + } } #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] @@ -178,7 +202,7 @@ impl PhysicalPlanBuilder { window: &crate::plans::Window, mut required: ColumnSet, _stat_info: PlanStatsInfo, - ) -> Result { + ) -> Result> { // 1. DO NOT Prune unused Columns cause window may not in required, eg: // select s1.a from ( select t1.a as a, dense_rank() over(order by t1.a desc) as rk // from (select 'a1' as a) t1 ) s1 @@ -382,8 +406,7 @@ impl PhysicalPlanBuilder { WindowFuncType::CumeDist => WindowFunction::CumeDist, }; - Ok(PhysicalPlan::Window(Window { - plan_id: 0, + Ok(Box::new(Window { index: w.index, input: Box::new(input), func, @@ -391,6 +414,7 @@ impl PhysicalPlanBuilder { order_by: order_by_items, window_frame: w.frame.clone(), limit: w.limit, + meta: PhysicalPlanMeta::new("Window"), })) } } diff --git a/src/query/sql/src/executor/physical_plans/physical_window_partition.rs b/src/query/sql/src/executor/physical_plans/physical_window_partition.rs index d26a5f4084a99..f24282e4466b8 100644 --- a/src/query/sql/src/executor/physical_plans/physical_window_partition.rs +++ b/src/query/sql/src/executor/physical_plans/physical_window_partition.rs @@ -23,8 +23,7 @@ use crate::IndexType; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct WindowPartition { - pub plan_id: u32, - meta: PhysicalPlanMeta, + pub meta: PhysicalPlanMeta, pub input: Box, pub partition_by: Vec, pub order_by: Vec, @@ -34,6 +33,7 @@ pub struct WindowPartition { pub stat_info: Option, } +#[typetag::serde] impl IPhysicalPlan for WindowPartition { fn get_meta(&self) -> &PhysicalPlanMeta { &self.meta @@ -47,7 +47,7 @@ impl IPhysicalPlan for WindowPartition { Box::new(std::iter::once(&self.input)) } - fn children_mut<'a>(&'a self) -> Box> + 'a> { + fn children_mut<'a>(&'a mut self) -> Box> + 'a> { Box::new(std::iter::once(&mut self.input)) } From 5b53eb0c34bac40bcbed93f19bde4e4e699116e5 Mon Sep 17 00:00:00 2001 From: zhang2014 Date: Mon, 30 Jun 2025 23:20:27 +0800 Subject: [PATCH 4/8] refactor(query): use trait to refactor physical plan --- .../src/schedulers/fragments/plan_fragment.rs | 109 +- src/query/sql/src/executor/mod.rs | 1 + src/query/sql/src/executor/physical_plan.rs | 18 +- .../sql/src/executor/physical_plan_visitor.rs | 1154 ++++++++--------- .../physical_add_stream_column.rs | 17 +- .../physical_aggregate_expand.rs | 15 + .../physical_aggregate_final.rs | 15 + .../physical_aggregate_partial.rs | 15 + .../physical_plans/physical_async_func.rs | 15 + .../physical_plans/physical_broadcast.rs | 26 + .../physical_plans/physical_cache_scan.rs | 11 + .../physical_column_mutation.rs | 16 +- .../physical_plans/physical_commit_sink.rs | 15 + .../physical_plans/physical_compact_source.rs | 11 + .../physical_constant_table_scan.rs | 11 + .../physical_copy_into_location.rs | 15 + .../physical_copy_into_table.rs | 16 +- .../physical_distributed_insert_select.rs | 15 + .../physical_plans/physical_eval_scalar.rs | 16 +- .../physical_plans/physical_exchange.rs | 15 + .../physical_plans/physical_exchange_sink.rs | 15 + .../physical_exchange_source.rs | 11 + .../physical_expression_scan.rs | 15 + .../physical_plans/physical_filter.rs | 15 + .../physical_plans/physical_hash_join.rs | 17 + .../executor/physical_plans/physical_limit.rs | 15 + .../physical_multi_table_insert.rs | 128 ++ .../physical_plans/physical_mutation.rs | 16 +- .../physical_mutation_into_organize.rs | 16 +- .../physical_mutation_into_split.rs | 16 +- .../physical_mutation_manipulate.rs | 16 +- .../physical_mutation_source.rs | 11 + .../physical_plans/physical_project_set.rs | 15 + .../physical_plans/physical_r_cte_scan.rs | 11 + .../physical_plans/physical_range_join.rs | 17 + .../physical_plans/physical_recluster.rs | 25 + .../physical_replace_async_source.rs | 11 + .../physical_replace_deduplicate.rs | 15 + .../physical_plans/physical_replace_into.rs | 15 + .../physical_plans/physical_row_fetch.rs | 15 + .../executor/physical_plans/physical_sort.rs | 15 + .../physical_plans/physical_table_scan.rs | 11 + .../executor/physical_plans/physical_udf.rs | 15 + .../physical_plans/physical_union_all.rs | 17 + .../physical_plans/physical_window.rs | 15 + .../physical_window_partition.rs | 15 + 46 files changed, 1396 insertions(+), 633 deletions(-) diff --git a/src/query/service/src/schedulers/fragments/plan_fragment.rs b/src/query/service/src/schedulers/fragments/plan_fragment.rs index 18f2b35267eb4..a3043937a0ed5 100644 --- a/src/query/service/src/schedulers/fragments/plan_fragment.rs +++ b/src/query/service/src/schedulers/fragments/plan_fragment.rs @@ -33,6 +33,7 @@ use databend_common_sql::executor::physical_plans::Recluster; use databend_common_sql::executor::physical_plans::ReplaceDeduplicate; use databend_common_sql::executor::physical_plans::ReplaceInto; use databend_common_sql::executor::physical_plans::TableScan; +use databend_common_sql::executor::{IPhysicalPlan, PhysicalPlanDeriveHandle}; use databend_common_storages_fuse::TableContext; use databend_storages_common_table_meta::meta::BlockSlotDescription; use databend_storages_common_table_meta::meta::Location; @@ -68,7 +69,7 @@ pub enum FragmentType { #[derive(Clone)] pub struct PlanFragment { - pub plan: PhysicalPlan, + pub plan: Box, pub fragment_type: FragmentType, pub fragment_id: usize, pub exchange: Option, @@ -276,13 +277,8 @@ impl PlanFragment { let mut plan = self.plan.clone(); let need_insert = executor == local_id; - let mut replace_replace_into = ReplaceReplaceInto { - partitions: parts, - slot: None, - need_insert, - }; - plan = replace_replace_into.replace(&plan)?; - + let mut handle = ReplaceDeriveHandle::new(parts, None, need_insert); + plan = plan.derive_with(&mut handle); fragment_actions.add_action(QueryFragmentAction::create(executor, plan)); } } @@ -293,15 +289,16 @@ impl PlanFragment { for (executor_idx, executor) in executors.into_iter().enumerate() { let mut plan = self.plan.clone(); let need_insert = executor == local_id; - let mut replace_replace_into = ReplaceReplaceInto { - partitions: partitions.clone(), - slot: Some(BlockSlotDescription { + let mut handle = ReplaceDeriveHandle::new( + partitions.clone(), + Some(BlockSlotDescription { num_slots, slot: executor_idx as u32, }), need_insert, - }; - plan = replace_replace_into.replace(&plan)?; + ); + + plan = plan.derive_with(&mut handle); fragment_actions.add_action(QueryFragmentAction::create(executor, plan)); } @@ -332,9 +329,8 @@ impl PlanFragment { for (executor, parts) in partition_reshuffle.into_iter() { let mut plan = self.plan.clone(); - let mut replace_compact_source = ReplaceCompactBlock { partitions: parts }; - plan = replace_compact_source.replace(&plan)?; - + let mut handle = CompactSourceDeriveHandle::new(parts); + plan = plan.derive_with(&mut handle); fragment_actions.add_action(QueryFragmentAction::create(executor, plan)); } @@ -577,47 +573,78 @@ impl PhysicalPlanReplacer for ReplaceMutationSource { } } -struct ReplaceCompactBlock { +struct CompactSourceDeriveHandle { pub partitions: Partitions, } -impl PhysicalPlanReplacer for ReplaceCompactBlock { - fn replace_compact_source(&mut self, plan: &CompactSource) -> Result { - Ok(PhysicalPlan::CompactSource(Box::new(CompactSource { +impl CompactSourceDeriveHandle { + pub fn new(partitions: Partitions) -> Box { + Box::new(CompactSourceDeriveHandle { + partitions + }) + } +} + +impl PhysicalPlanDeriveHandle for CompactSourceDeriveHandle { + fn derive( + &mut self, + v: &Box, + children: Vec>, + ) -> std::result::Result, Vec>> { + let Some(compact_source) = v.down_cast::() else { + return Err(children); + }; + + + Ok(Box::new(CompactSource { parts: self.partitions.clone(), - ..plan.clone() - }))) + ..compact_source.clone() + })) } } -struct ReplaceReplaceInto { +struct ReplaceDeriveHandle { pub partitions: Vec<(usize, Location)>, // for standalone mode, slot is None pub slot: Option, pub need_insert: bool, } -impl PhysicalPlanReplacer for ReplaceReplaceInto { - fn replace_replace_into(&mut self, plan: &ReplaceInto) -> Result { - let input = self.replace(&plan.input)?; - Ok(PhysicalPlan::ReplaceInto(Box::new(ReplaceInto { - input: Box::new(input), - need_insert: self.need_insert, - segments: self.partitions.clone(), - block_slots: self.slot.clone(), - ..plan.clone() - }))) +impl ReplaceDeriveHandle { + pub fn new(partitions: Vec<(usize, Location)>, slot: Option, need_insert: bool) -> Box { + Box::new(ReplaceDeriveHandle { + partitions, + slot, + need_insert, + }) } +} - fn replace_deduplicate(&mut self, plan: &ReplaceDeduplicate) -> Result { - let input = self.replace(&plan.input)?; - Ok(PhysicalPlan::ReplaceDeduplicate(Box::new( - ReplaceDeduplicate { - input: Box::new(input), +impl PhysicalPlanDeriveHandle for ReplaceDeriveHandle { + fn derive( + &mut self, + v: &Box, + children: Vec>, + ) -> std::result::Result, Vec>> { + if let Some(replace_into) = v.down_cast::() { + assert_eq!(children.len(), 1); + return Ok(Box::new(ReplaceInto { + input: children[0], + need_insert: self.need_insert, + segments: self.partitions.clone(), + block_slots: self.slot.clone(), + ..replace_into.clone() + })); + } else if let Some(replace_deduplicate) = v.down_cast::() { + assert_eq!(children.len(), 1); + return Ok(Box::new(ReplaceDeduplicate { + input: children[0], need_insert: self.need_insert, table_is_empty: self.partitions.is_empty(), - ..plan.clone() - }, - ))) + ..replace_deduplicate.clone() + })); + } + + None } } diff --git a/src/query/sql/src/executor/mod.rs b/src/query/sql/src/executor/mod.rs index d9aa9b341a1d7..13ae7a9c6f1c4 100644 --- a/src/query/sql/src/executor/mod.rs +++ b/src/query/sql/src/executor/mod.rs @@ -33,3 +33,4 @@ pub use physical_plans::build_broadcast_plans; pub use physical_plans::PhysicalRuntimeFilter; pub use physical_plans::PhysicalRuntimeFilters; pub use util::*; +pub use physical_plan::PhysicalPlanDeriveHandle; diff --git a/src/query/sql/src/executor/physical_plan.rs b/src/query/sql/src/executor/physical_plan.rs index a6d8fc2fe3047..28c5d33bc0fde 100644 --- a/src/query/sql/src/executor/physical_plan.rs +++ b/src/query/sql/src/executor/physical_plan.rs @@ -92,6 +92,14 @@ impl PhysicalPlanMeta { } } +pub trait PhysicalPlanDeriveHandle { + fn derive( + &mut self, + v: &Box, + children: Vec>, + ) -> std::result::Result, Vec>>; +} + #[typetag::serde] pub trait IPhysicalPlan: Debug { fn get_meta(&self) -> &PhysicalPlanMeta; @@ -184,12 +192,14 @@ pub trait IPhysicalPlan: Debug { fn get_labels(&self) -> Result>> { Ok(HashMap::new()) } + + fn derive_with(&self, handle: &mut Box) -> Box; } pub trait PhysicalPlanExt { fn clone_box(&self) -> Box; - fn visit(&self, visitor: Box) -> Result<()>; + fn down_cast(&self) -> Option<&T>; } impl PhysicalPlanExt for T @@ -199,12 +209,6 @@ where fn clone_box(&self) -> Box { Box::new(self.clone()) } - - fn visit(&self, visitor: Box) -> Result<()> { - visitor.downcast() - // visitor.downcast() - todo!() - } } impl Clone for Box { diff --git a/src/query/sql/src/executor/physical_plan_visitor.rs b/src/query/sql/src/executor/physical_plan_visitor.rs index cdfc27a60e28f..41d3bbcf1512d 100644 --- a/src/query/sql/src/executor/physical_plan_visitor.rs +++ b/src/query/sql/src/executor/physical_plan_visitor.rs @@ -24,7 +24,7 @@ use super::physical_plans::MutationManipulate; use super::physical_plans::MutationOrganize; use super::physical_plans::MutationSplit; use super::physical_plans::RecursiveCteScan; -use crate::executor::physical_plan::PhysicalPlan; +use crate::executor::physical_plan::{PhysicalPlan, PhysicalPlanExt}; use crate::executor::physical_plans::AggregateExpand; use crate::executor::physical_plans::AggregateFinal; use crate::executor::physical_plans::AggregatePartial; @@ -68,582 +68,582 @@ use crate::executor::physical_plans::Udf; use crate::executor::physical_plans::UnionAll; use crate::executor::physical_plans::Window; use crate::executor::physical_plans::WindowPartition; - -pub trait PhysicalPlanReplacer { - fn replace(&mut self, plan: &PhysicalPlan) -> Result { - match plan { - PhysicalPlan::TableScan(plan) => self.replace_table_scan(plan), - PhysicalPlan::RecursiveCteScan(plan) => self.replace_recursive_cte_scan(plan), - PhysicalPlan::Filter(plan) => self.replace_filter(plan), - PhysicalPlan::EvalScalar(plan) => self.replace_eval_scalar(plan), - PhysicalPlan::AggregateExpand(plan) => self.replace_aggregate_expand(plan), - PhysicalPlan::AggregatePartial(plan) => self.replace_aggregate_partial(plan), - PhysicalPlan::AggregateFinal(plan) => self.replace_aggregate_final(plan), - PhysicalPlan::Window(plan) => self.replace_window(plan), - PhysicalPlan::WindowPartition(plan) => self.replace_window_partition(plan), - PhysicalPlan::Sort(plan) => self.replace_sort(plan), - PhysicalPlan::Limit(plan) => self.replace_limit(plan), - PhysicalPlan::RowFetch(plan) => self.replace_row_fetch(plan), - PhysicalPlan::HashJoin(plan) => self.replace_hash_join(plan), - PhysicalPlan::Exchange(plan) => self.replace_exchange(plan), - PhysicalPlan::ExchangeSource(plan) => self.replace_exchange_source(plan), - PhysicalPlan::ExchangeSink(plan) => self.replace_exchange_sink(plan), - PhysicalPlan::UnionAll(plan) => self.replace_union(plan), - PhysicalPlan::DistributedInsertSelect(plan) => self.replace_insert_select(plan), - PhysicalPlan::ProjectSet(plan) => self.replace_project_set(plan), - PhysicalPlan::CompactSource(plan) => self.replace_compact_source(plan), - PhysicalPlan::CommitSink(plan) => self.replace_commit_sink(plan), - PhysicalPlan::RangeJoin(plan) => self.replace_range_join(plan), - PhysicalPlan::CopyIntoTable(plan) => self.replace_copy_into_table(plan), - PhysicalPlan::CopyIntoLocation(plan) => self.replace_copy_into_location(plan), - PhysicalPlan::ReplaceAsyncSourcer(plan) => self.replace_async_sourcer(plan), - PhysicalPlan::ReplaceDeduplicate(plan) => self.replace_deduplicate(plan), - PhysicalPlan::ReplaceInto(plan) => self.replace_replace_into(plan), - PhysicalPlan::MutationSource(plan) => self.replace_mutation_source(plan), - PhysicalPlan::ColumnMutation(plan) => self.replace_column_mutation(plan), - PhysicalPlan::Mutation(plan) => self.replace_mutation(plan), - PhysicalPlan::MutationSplit(plan) => self.replace_mutation_split(plan), - PhysicalPlan::MutationManipulate(plan) => self.replace_mutation_manipulate(plan), - PhysicalPlan::MutationOrganize(plan) => self.replace_mutation_organize(plan), - PhysicalPlan::AddStreamColumn(plan) => self.replace_add_stream_column(plan), - PhysicalPlan::ConstantTableScan(plan) => self.replace_constant_table_scan(plan), - PhysicalPlan::ExpressionScan(plan) => self.replace_expression_scan(plan), - PhysicalPlan::CacheScan(plan) => self.replace_cache_scan(plan), - PhysicalPlan::Recluster(plan) => self.replace_recluster(plan), - PhysicalPlan::HilbertPartition(plan) => self.replace_hilbert_serialize(plan), - PhysicalPlan::Udf(plan) => self.replace_udf(plan), - PhysicalPlan::AsyncFunction(plan) => self.replace_async_function(plan), - PhysicalPlan::Duplicate(plan) => self.replace_duplicate(plan), - PhysicalPlan::Shuffle(plan) => self.replace_shuffle(plan), - PhysicalPlan::ChunkFilter(plan) => self.replace_chunk_filter(plan), - PhysicalPlan::ChunkEvalScalar(plan) => self.replace_chunk_eval_scalar(plan), - PhysicalPlan::ChunkCastSchema(plan) => self.replace_chunk_cast_schema(plan), - PhysicalPlan::ChunkFillAndReorder(plan) => self.replace_chunk_fill_and_reorder(plan), - PhysicalPlan::ChunkAppendData(plan) => self.replace_chunk_append_data(plan), - PhysicalPlan::ChunkMerge(plan) => self.replace_chunk_merge(plan), - PhysicalPlan::ChunkCommitInsert(plan) => self.replace_chunk_commit_insert(plan), - PhysicalPlan::BroadcastSource(plan) => self.replace_runtime_filter_source(plan), - PhysicalPlan::BroadcastSink(plan) => self.replace_runtime_filter_sink(plan), - } - } - - fn replace_runtime_filter_source(&mut self, plan: &BroadcastSource) -> Result { - Ok(PhysicalPlan::BroadcastSource(plan.clone())) - } - - fn replace_runtime_filter_sink(&mut self, plan: &BroadcastSink) -> Result> { - let input = self.replace(&plan.input)?; - Ok(Box::new(BroadcastSink { - broadcast_id: plan.broadcast_id, - input: Box::new(input), - meta: PhysicalPlanMeta::new("BroadcastSink"), - })) - } - - fn replace_recluster(&mut self, plan: &Recluster) -> Result { - Ok(PhysicalPlan::Recluster(Box::new(plan.clone()))) - } - - fn replace_hilbert_serialize(&mut self, plan: &HilbertPartition) -> Result { - let input = self.replace(&plan.input)?; - Ok(PhysicalPlan::HilbertPartition(Box::new(HilbertPartition { - input: Box::new(input), - ..plan.clone() - }))) - } - - fn replace_table_scan(&mut self, plan: &TableScan) -> Result { - Ok(PhysicalPlan::TableScan(plan.clone())) - } - - fn replace_recursive_cte_scan(&mut self, plan: &RecursiveCteScan) -> Result { - Ok(PhysicalPlan::RecursiveCteScan(plan.clone())) - } - - fn replace_constant_table_scan(&mut self, plan: &ConstantTableScan) -> Result { - Ok(PhysicalPlan::ConstantTableScan(plan.clone())) - } - - fn replace_expression_scan(&mut self, plan: &ExpressionScan) -> Result { - Ok(PhysicalPlan::ExpressionScan(plan.clone())) - } - - fn replace_cache_scan(&mut self, plan: &CacheScan) -> Result { - Ok(PhysicalPlan::CacheScan(plan.clone())) - } - - fn replace_filter(&mut self, plan: &Filter) -> Result { - let input = self.replace(&plan.input)?; - - Ok(Box::new(Filter { - projections: plan.projections.clone(), - input: Box::new(input), - predicates: plan.predicates.clone(), - stat_info: plan.stat_info.clone(), - meta: PhysicalPlanMeta::new("Filter"), - })) - } - - fn replace_eval_scalar(&mut self, plan: &EvalScalar) -> Result { - let input = self.replace(&plan.input)?; - - Ok(PhysicalPlan::EvalScalar(EvalScalar { - plan_id: plan.plan_id, - projections: plan.projections.clone(), - input: Box::new(input), - exprs: plan.exprs.clone(), - stat_info: plan.stat_info.clone(), - })) - } - - fn replace_aggregate_expand(&mut self, plan: &AggregateExpand) -> Result { - let input = self.replace(&plan.input)?; - - Ok(PhysicalPlan::AggregateExpand(AggregateExpand { - plan_id: plan.plan_id, - input: Box::new(input), - group_bys: plan.group_bys.clone(), - grouping_sets: plan.grouping_sets.clone(), - stat_info: plan.stat_info.clone(), - })) - } - - fn replace_aggregate_partial(&mut self, plan: &AggregatePartial) -> Result { - let input = self.replace(&plan.input)?; - - Ok(PhysicalPlan::AggregatePartial(AggregatePartial { - plan_id: plan.plan_id, - input: Box::new(input), - enable_experimental_aggregate_hashtable: plan.enable_experimental_aggregate_hashtable, - group_by: plan.group_by.clone(), - group_by_display: plan.group_by_display.clone(), - agg_funcs: plan.agg_funcs.clone(), - stat_info: plan.stat_info.clone(), - rank_limit: plan.rank_limit.clone(), - })) - } - - fn replace_aggregate_final(&mut self, plan: &AggregateFinal) -> Result { - let input = self.replace(&plan.input)?; - - Ok(PhysicalPlan::AggregateFinal(AggregateFinal { - plan_id: plan.plan_id, - input: Box::new(input), - before_group_by_schema: plan.before_group_by_schema.clone(), - group_by: plan.group_by.clone(), - agg_funcs: plan.agg_funcs.clone(), - group_by_display: plan.group_by_display.clone(), - stat_info: plan.stat_info.clone(), - })) - } - - fn replace_window(&mut self, plan: &Window) -> Result { - let input = self.replace(&plan.input)?; - - Ok(PhysicalPlan::Window(Window { - plan_id: plan.plan_id, - index: plan.index, - input: Box::new(input), - func: plan.func.clone(), - partition_by: plan.partition_by.clone(), - order_by: plan.order_by.clone(), - window_frame: plan.window_frame.clone(), - limit: plan.limit, - })) - } - - fn replace_window_partition(&mut self, plan: &WindowPartition) -> Result { - let input = self.replace(&plan.input)?; - - Ok(PhysicalPlan::WindowPartition(WindowPartition { - plan_id: plan.plan_id, - input: Box::new(input), - partition_by: plan.partition_by.clone(), - order_by: plan.order_by.clone(), - after_exchange: plan.after_exchange, - top_n: plan.top_n.clone(), - stat_info: plan.stat_info.clone(), - })) - } - - fn replace_hash_join(&mut self, plan: &HashJoin) -> Result { - let build = self.replace(&plan.build)?; - let probe = self.replace(&plan.probe)?; - - Ok(PhysicalPlan::HashJoin(HashJoin { - plan_id: plan.plan_id, - projections: plan.projections.clone(), - probe_projections: plan.probe_projections.clone(), - build_projections: plan.build_projections.clone(), - build: Box::new(build), - probe: Box::new(probe), - build_keys: plan.build_keys.clone(), - probe_keys: plan.probe_keys.clone(), - is_null_equal: plan.is_null_equal.clone(), - non_equi_conditions: plan.non_equi_conditions.clone(), - join_type: plan.join_type.clone(), - marker_index: plan.marker_index, - from_correlated_subquery: plan.from_correlated_subquery, - probe_to_build: plan.probe_to_build.clone(), - output_schema: plan.output_schema.clone(), - need_hold_hash_table: plan.need_hold_hash_table, - stat_info: plan.stat_info.clone(), - runtime_filter: plan.runtime_filter.clone(), - single_to_inner: plan.single_to_inner.clone(), - build_side_cache_info: plan.build_side_cache_info.clone(), - broadcast_id: plan.broadcast_id, - })) - } - - fn replace_range_join(&mut self, plan: &RangeJoin) -> Result { - let left = self.replace(&plan.left)?; - let right = self.replace(&plan.right)?; - - Ok(PhysicalPlan::RangeJoin(RangeJoin { - plan_id: plan.plan_id, - left: Box::new(left), - right: Box::new(right), - conditions: plan.conditions.clone(), - other_conditions: plan.other_conditions.clone(), - join_type: plan.join_type.clone(), - range_join_type: plan.range_join_type.clone(), - output_schema: plan.output_schema.clone(), - stat_info: plan.stat_info.clone(), - })) - } - - fn replace_sort(&mut self, plan: &Sort) -> Result { - let input = self.replace(&plan.input)?; - - Ok(PhysicalPlan::Sort(Sort { - plan_id: plan.plan_id, - input: Box::new(input), - order_by: plan.order_by.clone(), - limit: plan.limit, - after_exchange: plan.after_exchange, - pre_projection: plan.pre_projection.clone(), - stat_info: plan.stat_info.clone(), - })) - } - - fn replace_limit(&mut self, plan: &Limit) -> Result { - let input = self.replace(&plan.input)?; - - Ok(PhysicalPlan::Limit(Limit { - plan_id: plan.plan_id, - input: Box::new(input), - limit: plan.limit, - offset: plan.offset, - stat_info: plan.stat_info.clone(), - })) - } - - fn replace_row_fetch(&mut self, plan: &RowFetch) -> Result { - let input = self.replace(&plan.input)?; - - Ok(PhysicalPlan::RowFetch(RowFetch { - plan_id: plan.plan_id, - input: Box::new(input), - source: plan.source.clone(), - row_id_col_offset: plan.row_id_col_offset, - cols_to_fetch: plan.cols_to_fetch.clone(), - fetched_fields: plan.fetched_fields.clone(), - stat_info: plan.stat_info.clone(), - need_wrap_nullable: plan.need_wrap_nullable, - })) - } - - fn replace_exchange(&mut self, plan: &Exchange) -> Result { - let input = self.replace(&plan.input)?; - - Ok(PhysicalPlan::Exchange(Exchange { - plan_id: plan.plan_id, - input: Box::new(input), - kind: plan.kind.clone(), - keys: plan.keys.clone(), - ignore_exchange: plan.ignore_exchange, - allow_adjust_parallelism: plan.allow_adjust_parallelism, - })) - } - - fn replace_exchange_source(&mut self, plan: &ExchangeSource) -> Result { - Ok(PhysicalPlan::ExchangeSource(plan.clone())) - } - - fn replace_exchange_sink(&mut self, plan: &ExchangeSink) -> Result { - let input = self.replace(&plan.input)?; - - Ok(PhysicalPlan::ExchangeSink(ExchangeSink { - // TODO(leiysky): we reuse the plan id of the Exchange node here, - // should generate a new one. - plan_id: plan.plan_id, - - input: Box::new(input), - schema: plan.schema.clone(), - kind: plan.kind.clone(), - keys: plan.keys.clone(), - destination_fragment_id: plan.destination_fragment_id, - query_id: plan.query_id.clone(), - ignore_exchange: plan.ignore_exchange, - allow_adjust_parallelism: plan.allow_adjust_parallelism, - })) - } - - fn replace_union(&mut self, plan: &UnionAll) -> Result { - let left = self.replace(&plan.left)?; - let right = self.replace(&plan.right)?; - Ok(PhysicalPlan::UnionAll(UnionAll { - plan_id: plan.plan_id, - left: Box::new(left), - right: Box::new(right), - left_outputs: plan.left_outputs.clone(), - right_outputs: plan.right_outputs.clone(), - schema: plan.schema.clone(), - stat_info: plan.stat_info.clone(), - cte_scan_names: plan.cte_scan_names.clone(), - })) - } - - fn replace_copy_into_table(&mut self, plan: &CopyIntoTable) -> Result { - match &plan.source { - CopyIntoTableSource::Stage(_) => { - Ok(PhysicalPlan::CopyIntoTable(Box::new(plan.clone()))) - } - CopyIntoTableSource::Query(query_physical_plan) => { - let input = self.replace(query_physical_plan)?; - Ok(PhysicalPlan::CopyIntoTable(Box::new(CopyIntoTable { - source: CopyIntoTableSource::Query(Box::new(input)), - ..plan.clone() - }))) - } - } - } - - fn replace_copy_into_location(&mut self, plan: &CopyIntoLocation) -> Result { - let input = self.replace(&plan.input)?; - - Ok(PhysicalPlan::CopyIntoLocation(Box::new(CopyIntoLocation { - plan_id: plan.plan_id, - input: Box::new(input), - project_columns: plan.project_columns.clone(), - input_data_schema: plan.input_data_schema.clone(), - input_table_schema: plan.input_table_schema.clone(), - info: plan.info.clone(), - }))) - } - - fn replace_insert_select(&mut self, plan: &DistributedInsertSelect) -> Result { - let input = self.replace(&plan.input)?; - - Ok(PhysicalPlan::DistributedInsertSelect(Box::new( - DistributedInsertSelect { - input: Box::new(input), - ..plan.clone() - }, - ))) - } - - fn replace_compact_source(&mut self, plan: &CompactSource) -> Result { - Ok(PhysicalPlan::CompactSource(Box::new(plan.clone()))) - } - - fn replace_commit_sink(&mut self, plan: &CommitSink) -> Result { - let input = self.replace(&plan.input)?; - Ok(PhysicalPlan::CommitSink(Box::new(CommitSink { - input: Box::new(input), - ..plan.clone() - }))) - } - - fn replace_async_sourcer(&mut self, plan: &ReplaceAsyncSourcer) -> Result { - Ok(PhysicalPlan::ReplaceAsyncSourcer(plan.clone())) - } - - fn replace_deduplicate(&mut self, plan: &ReplaceDeduplicate) -> Result { - let input = self.replace(&plan.input)?; - Ok(PhysicalPlan::ReplaceDeduplicate(Box::new( - ReplaceDeduplicate { - input: Box::new(input), - ..plan.clone() - }, - ))) - } - - fn replace_replace_into(&mut self, plan: &ReplaceInto) -> Result { - let input = self.replace(&plan.input)?; - Ok(PhysicalPlan::ReplaceInto(Box::new(ReplaceInto { - input: Box::new(input), - ..plan.clone() - }))) - } - - fn replace_mutation_source(&mut self, plan: &MutationSource) -> Result { - Ok(PhysicalPlan::MutationSource(plan.clone())) - } - - fn replace_column_mutation(&mut self, plan: &ColumnMutation) -> Result { - let input = self.replace(&plan.input)?; - Ok(PhysicalPlan::ColumnMutation(ColumnMutation { - input: Box::new(input), - ..plan.clone() - })) - } - - fn replace_mutation(&mut self, plan: &Mutation) -> Result { - let input = self.replace(&plan.input)?; - Ok(PhysicalPlan::Mutation(Box::new(Mutation { - input: Box::new(input), - ..plan.clone() - }))) - } - - fn replace_mutation_split(&mut self, plan: &MutationSplit) -> Result { - let input = self.replace(&plan.input)?; - Ok(PhysicalPlan::MutationSplit(Box::new(MutationSplit { - input: Box::new(input), - ..plan.clone() - }))) - } - - fn replace_mutation_manipulate(&mut self, plan: &MutationManipulate) -> Result { - let input = self.replace(&plan.input)?; - Ok(PhysicalPlan::MutationManipulate(Box::new( - MutationManipulate { - input: Box::new(input), - ..plan.clone() - }, - ))) - } - - fn replace_mutation_organize(&mut self, plan: &MutationOrganize) -> Result { - let input = self.replace(&plan.input)?; - Ok(PhysicalPlan::MutationOrganize(Box::new(MutationOrganize { - input: Box::new(input), - ..plan.clone() - }))) - } - - fn replace_add_stream_column(&mut self, plan: &AddStreamColumn) -> Result { - let input = self.replace(&plan.input)?; - Ok(PhysicalPlan::AddStreamColumn(Box::new(AddStreamColumn { - input: Box::new(input), - ..plan.clone() - }))) - } - - fn replace_project_set(&mut self, plan: &ProjectSet) -> Result { - let input = self.replace(&plan.input)?; - Ok(PhysicalPlan::ProjectSet(ProjectSet { - plan_id: plan.plan_id, - input: Box::new(input), - srf_exprs: plan.srf_exprs.clone(), - projections: plan.projections.clone(), - stat_info: plan.stat_info.clone(), - })) - } - - fn replace_udf(&mut self, plan: &Udf) -> Result { - let input = self.replace(&plan.input)?; - Ok(PhysicalPlan::Udf(Udf { - plan_id: plan.plan_id, - input: Box::new(input), - udf_funcs: plan.udf_funcs.clone(), - stat_info: plan.stat_info.clone(), - script_udf: plan.script_udf, - })) - } - - fn replace_async_function(&mut self, plan: &AsyncFunction) -> Result { - let input = self.replace(&plan.input)?; - Ok(PhysicalPlan::AsyncFunction(AsyncFunction { - plan_id: plan.plan_id, - input: Box::new(input), - async_func_descs: plan.async_func_descs.clone(), - stat_info: plan.stat_info.clone(), - })) - } - - fn replace_duplicate(&mut self, plan: &Duplicate) -> Result { - let input = self.replace(&plan.input)?; - Ok(PhysicalPlan::Duplicate(Box::new(Duplicate { - input: Box::new(input), - ..plan.clone() - }))) - } - - fn replace_shuffle(&mut self, plan: &Shuffle) -> Result { - let input = self.replace(&plan.input)?; - Ok(PhysicalPlan::Shuffle(Box::new(Shuffle { - input: Box::new(input), - ..plan.clone() - }))) - } - - fn replace_chunk_filter(&mut self, plan: &ChunkFilter) -> Result { - let input = self.replace(&plan.input)?; - Ok(PhysicalPlan::ChunkFilter(Box::new(ChunkFilter { - input: Box::new(input), - predicates: plan.predicates.clone(), - ..plan.clone() - }))) - } - - fn replace_chunk_eval_scalar(&mut self, plan: &ChunkEvalScalar) -> Result { - let input = self.replace(&plan.input)?; - Ok(PhysicalPlan::ChunkEvalScalar(Box::new(ChunkEvalScalar { - input: Box::new(input), - ..plan.clone() - }))) - } - - fn replace_chunk_cast_schema(&mut self, plan: &ChunkCastSchema) -> Result { - let input = self.replace(&plan.input)?; - Ok(PhysicalPlan::ChunkCastSchema(Box::new(ChunkCastSchema { - input: Box::new(input), - cast_schemas: plan.cast_schemas.clone(), - ..plan.clone() - }))) - } - - fn replace_chunk_fill_and_reorder( - &mut self, - plan: &ChunkFillAndReorder, - ) -> Result { - let input = self.replace(&plan.input)?; - Ok(PhysicalPlan::ChunkFillAndReorder(Box::new( - ChunkFillAndReorder { - input: Box::new(input), - ..plan.clone() - }, - ))) - } - - fn replace_chunk_append_data(&mut self, plan: &ChunkAppendData) -> Result { - let input = self.replace(&plan.input)?; - Ok(PhysicalPlan::ChunkAppendData(Box::new(ChunkAppendData { - input: Box::new(input), - ..plan.clone() - }))) - } - - fn replace_chunk_merge(&mut self, plan: &ChunkMerge) -> Result { - let input = self.replace(&plan.input)?; - Ok(PhysicalPlan::ChunkMerge(Box::new(ChunkMerge { - input: Box::new(input), - ..plan.clone() - }))) - } - - fn replace_chunk_commit_insert(&mut self, plan: &ChunkCommitInsert) -> Result { - let input = self.replace(&plan.input)?; - Ok(PhysicalPlan::ChunkCommitInsert(Box::new( - ChunkCommitInsert { - input: Box::new(input), - ..plan.clone() - }, - ))) - } -} +// +// pub trait PhysicalPlanReplacer { +// fn replace(&mut self, plan: &PhysicalPlan) -> Result { +// match plan { +// PhysicalPlan::TableScan(plan) => self.replace_table_scan(plan), +// PhysicalPlan::RecursiveCteScan(plan) => self.replace_recursive_cte_scan(plan), +// PhysicalPlan::Filter(plan) => self.replace_filter(plan), +// PhysicalPlan::EvalScalar(plan) => self.replace_eval_scalar(plan), +// PhysicalPlan::AggregateExpand(plan) => self.replace_aggregate_expand(plan), +// PhysicalPlan::AggregatePartial(plan) => self.replace_aggregate_partial(plan), +// PhysicalPlan::AggregateFinal(plan) => self.replace_aggregate_final(plan), +// PhysicalPlan::Window(plan) => self.replace_window(plan), +// PhysicalPlan::WindowPartition(plan) => self.replace_window_partition(plan), +// PhysicalPlan::Sort(plan) => self.replace_sort(plan), +// PhysicalPlan::Limit(plan) => self.replace_limit(plan), +// PhysicalPlan::RowFetch(plan) => self.replace_row_fetch(plan), +// PhysicalPlan::HashJoin(plan) => self.replace_hash_join(plan), +// PhysicalPlan::Exchange(plan) => self.replace_exchange(plan), +// PhysicalPlan::ExchangeSource(plan) => self.replace_exchange_source(plan), +// PhysicalPlan::ExchangeSink(plan) => self.replace_exchange_sink(plan), +// PhysicalPlan::UnionAll(plan) => self.replace_union(plan), +// PhysicalPlan::DistributedInsertSelect(plan) => self.replace_insert_select(plan), +// PhysicalPlan::ProjectSet(plan) => self.replace_project_set(plan), +// PhysicalPlan::CompactSource(plan) => self.replace_compact_source(plan), +// PhysicalPlan::CommitSink(plan) => self.replace_commit_sink(plan), +// PhysicalPlan::RangeJoin(plan) => self.replace_range_join(plan), +// PhysicalPlan::CopyIntoTable(plan) => self.replace_copy_into_table(plan), +// PhysicalPlan::CopyIntoLocation(plan) => self.replace_copy_into_location(plan), +// PhysicalPlan::ReplaceAsyncSourcer(plan) => self.replace_async_sourcer(plan), +// PhysicalPlan::ReplaceDeduplicate(plan) => self.replace_deduplicate(plan), +// PhysicalPlan::ReplaceInto(plan) => self.replace_replace_into(plan), +// PhysicalPlan::MutationSource(plan) => self.replace_mutation_source(plan), +// PhysicalPlan::ColumnMutation(plan) => self.replace_column_mutation(plan), +// PhysicalPlan::Mutation(plan) => self.replace_mutation(plan), +// PhysicalPlan::MutationSplit(plan) => self.replace_mutation_split(plan), +// PhysicalPlan::MutationManipulate(plan) => self.replace_mutation_manipulate(plan), +// PhysicalPlan::MutationOrganize(plan) => self.replace_mutation_organize(plan), +// PhysicalPlan::AddStreamColumn(plan) => self.replace_add_stream_column(plan), +// PhysicalPlan::ConstantTableScan(plan) => self.replace_constant_table_scan(plan), +// PhysicalPlan::ExpressionScan(plan) => self.replace_expression_scan(plan), +// PhysicalPlan::CacheScan(plan) => self.replace_cache_scan(plan), +// PhysicalPlan::Recluster(plan) => self.replace_recluster(plan), +// PhysicalPlan::HilbertPartition(plan) => self.replace_hilbert_serialize(plan), +// PhysicalPlan::Udf(plan) => self.replace_udf(plan), +// PhysicalPlan::AsyncFunction(plan) => self.replace_async_function(plan), +// PhysicalPlan::Duplicate(plan) => self.replace_duplicate(plan), +// PhysicalPlan::Shuffle(plan) => self.replace_shuffle(plan), +// PhysicalPlan::ChunkFilter(plan) => self.replace_chunk_filter(plan), +// PhysicalPlan::ChunkEvalScalar(plan) => self.replace_chunk_eval_scalar(plan), +// PhysicalPlan::ChunkCastSchema(plan) => self.replace_chunk_cast_schema(plan), +// PhysicalPlan::ChunkFillAndReorder(plan) => self.replace_chunk_fill_and_reorder(plan), +// PhysicalPlan::ChunkAppendData(plan) => self.replace_chunk_append_data(plan), +// PhysicalPlan::ChunkMerge(plan) => self.replace_chunk_merge(plan), +// PhysicalPlan::ChunkCommitInsert(plan) => self.replace_chunk_commit_insert(plan), +// PhysicalPlan::BroadcastSource(plan) => self.replace_runtime_filter_source(plan), +// PhysicalPlan::BroadcastSink(plan) => self.replace_runtime_filter_sink(plan), +// } +// } +// +// fn replace_runtime_filter_source(&mut self, plan: &BroadcastSource) -> Result { +// Ok(PhysicalPlan::BroadcastSource(plan.clone())) +// } +// +// fn replace_runtime_filter_sink(&mut self, plan: &BroadcastSink) -> Result> { +// let input = self.replace(&plan.input)?; +// Ok(Box::new(BroadcastSink { +// broadcast_id: plan.broadcast_id, +// input: Box::new(input), +// meta: PhysicalPlanMeta::new("BroadcastSink"), +// })) +// } +// +// fn replace_recluster(&mut self, plan: &Recluster) -> Result { +// Ok(PhysicalPlan::Recluster(Box::new(plan.clone()))) +// } +// +// fn replace_hilbert_serialize(&mut self, plan: &HilbertPartition) -> Result { +// let input = self.replace(&plan.input)?; +// Ok(PhysicalPlan::HilbertPartition(Box::new(HilbertPartition { +// input: Box::new(input), +// ..plan.clone() +// }))) +// } +// +// fn replace_table_scan(&mut self, plan: &TableScan) -> Result { +// Ok(PhysicalPlan::TableScan(plan.clone())) +// } +// +// fn replace_recursive_cte_scan(&mut self, plan: &RecursiveCteScan) -> Result { +// Ok(PhysicalPlan::RecursiveCteScan(plan.clone())) +// } +// +// fn replace_constant_table_scan(&mut self, plan: &ConstantTableScan) -> Result { +// Ok(PhysicalPlan::ConstantTableScan(plan.clone())) +// } +// +// fn replace_expression_scan(&mut self, plan: &ExpressionScan) -> Result { +// Ok(PhysicalPlan::ExpressionScan(plan.clone())) +// } +// +// fn replace_cache_scan(&mut self, plan: &CacheScan) -> Result { +// Ok(PhysicalPlan::CacheScan(plan.clone())) +// } +// +// fn replace_filter(&mut self, plan: &Filter) -> Result { +// let input = self.replace(&plan.input)?; +// +// Ok(Box::new(Filter { +// projections: plan.projections.clone(), +// input: Box::new(input), +// predicates: plan.predicates.clone(), +// stat_info: plan.stat_info.clone(), +// meta: PhysicalPlanMeta::new("Filter"), +// })) +// } +// +// fn replace_eval_scalar(&mut self, plan: &EvalScalar) -> Result { +// let input = self.replace(&plan.input)?; +// +// Ok(PhysicalPlan::EvalScalar(EvalScalar { +// plan_id: plan.plan_id, +// projections: plan.projections.clone(), +// input: Box::new(input), +// exprs: plan.exprs.clone(), +// stat_info: plan.stat_info.clone(), +// })) +// } +// +// fn replace_aggregate_expand(&mut self, plan: &AggregateExpand) -> Result { +// let input = self.replace(&plan.input)?; +// +// Ok(PhysicalPlan::AggregateExpand(AggregateExpand { +// plan_id: plan.plan_id, +// input: Box::new(input), +// group_bys: plan.group_bys.clone(), +// grouping_sets: plan.grouping_sets.clone(), +// stat_info: plan.stat_info.clone(), +// })) +// } +// +// fn replace_aggregate_partial(&mut self, plan: &AggregatePartial) -> Result { +// let input = self.replace(&plan.input)?; +// +// Ok(PhysicalPlan::AggregatePartial(AggregatePartial { +// plan_id: plan.plan_id, +// input: Box::new(input), +// enable_experimental_aggregate_hashtable: plan.enable_experimental_aggregate_hashtable, +// group_by: plan.group_by.clone(), +// group_by_display: plan.group_by_display.clone(), +// agg_funcs: plan.agg_funcs.clone(), +// stat_info: plan.stat_info.clone(), +// rank_limit: plan.rank_limit.clone(), +// })) +// } +// +// fn replace_aggregate_final(&mut self, plan: &AggregateFinal) -> Result { +// let input = self.replace(&plan.input)?; +// +// Ok(PhysicalPlan::AggregateFinal(AggregateFinal { +// plan_id: plan.plan_id, +// input: Box::new(input), +// before_group_by_schema: plan.before_group_by_schema.clone(), +// group_by: plan.group_by.clone(), +// agg_funcs: plan.agg_funcs.clone(), +// group_by_display: plan.group_by_display.clone(), +// stat_info: plan.stat_info.clone(), +// })) +// } +// +// fn replace_window(&mut self, plan: &Window) -> Result { +// let input = self.replace(&plan.input)?; +// +// Ok(PhysicalPlan::Window(Window { +// plan_id: plan.plan_id, +// index: plan.index, +// input: Box::new(input), +// func: plan.func.clone(), +// partition_by: plan.partition_by.clone(), +// order_by: plan.order_by.clone(), +// window_frame: plan.window_frame.clone(), +// limit: plan.limit, +// })) +// } +// +// fn replace_window_partition(&mut self, plan: &WindowPartition) -> Result { +// let input = self.replace(&plan.input)?; +// +// Ok(PhysicalPlan::WindowPartition(WindowPartition { +// plan_id: plan.plan_id, +// input: Box::new(input), +// partition_by: plan.partition_by.clone(), +// order_by: plan.order_by.clone(), +// after_exchange: plan.after_exchange, +// top_n: plan.top_n.clone(), +// stat_info: plan.stat_info.clone(), +// })) +// } +// +// fn replace_hash_join(&mut self, plan: &HashJoin) -> Result { +// let build = self.replace(&plan.build)?; +// let probe = self.replace(&plan.probe)?; +// +// Ok(PhysicalPlan::HashJoin(HashJoin { +// plan_id: plan.plan_id, +// projections: plan.projections.clone(), +// probe_projections: plan.probe_projections.clone(), +// build_projections: plan.build_projections.clone(), +// build: Box::new(build), +// probe: Box::new(probe), +// build_keys: plan.build_keys.clone(), +// probe_keys: plan.probe_keys.clone(), +// is_null_equal: plan.is_null_equal.clone(), +// non_equi_conditions: plan.non_equi_conditions.clone(), +// join_type: plan.join_type.clone(), +// marker_index: plan.marker_index, +// from_correlated_subquery: plan.from_correlated_subquery, +// probe_to_build: plan.probe_to_build.clone(), +// output_schema: plan.output_schema.clone(), +// need_hold_hash_table: plan.need_hold_hash_table, +// stat_info: plan.stat_info.clone(), +// runtime_filter: plan.runtime_filter.clone(), +// single_to_inner: plan.single_to_inner.clone(), +// build_side_cache_info: plan.build_side_cache_info.clone(), +// broadcast_id: plan.broadcast_id, +// })) +// } +// +// fn replace_range_join(&mut self, plan: &RangeJoin) -> Result { +// let left = self.replace(&plan.left)?; +// let right = self.replace(&plan.right)?; +// +// Ok(PhysicalPlan::RangeJoin(RangeJoin { +// plan_id: plan.plan_id, +// left: Box::new(left), +// right: Box::new(right), +// conditions: plan.conditions.clone(), +// other_conditions: plan.other_conditions.clone(), +// join_type: plan.join_type.clone(), +// range_join_type: plan.range_join_type.clone(), +// output_schema: plan.output_schema.clone(), +// stat_info: plan.stat_info.clone(), +// })) +// } +// +// fn replace_sort(&mut self, plan: &Sort) -> Result { +// let input = self.replace(&plan.input)?; +// +// Ok(PhysicalPlan::Sort(Sort { +// plan_id: plan.plan_id, +// input: Box::new(input), +// order_by: plan.order_by.clone(), +// limit: plan.limit, +// after_exchange: plan.after_exchange, +// pre_projection: plan.pre_projection.clone(), +// stat_info: plan.stat_info.clone(), +// })) +// } +// +// fn replace_limit(&mut self, plan: &Limit) -> Result { +// let input = self.replace(&plan.input)?; +// +// Ok(PhysicalPlan::Limit(Limit { +// plan_id: plan.plan_id, +// input: Box::new(input), +// limit: plan.limit, +// offset: plan.offset, +// stat_info: plan.stat_info.clone(), +// })) +// } +// +// fn replace_row_fetch(&mut self, plan: &RowFetch) -> Result { +// let input = self.replace(&plan.input)?; +// +// Ok(PhysicalPlan::RowFetch(RowFetch { +// plan_id: plan.plan_id, +// input: Box::new(input), +// source: plan.source.clone(), +// row_id_col_offset: plan.row_id_col_offset, +// cols_to_fetch: plan.cols_to_fetch.clone(), +// fetched_fields: plan.fetched_fields.clone(), +// stat_info: plan.stat_info.clone(), +// need_wrap_nullable: plan.need_wrap_nullable, +// })) +// } +// +// fn replace_exchange(&mut self, plan: &Exchange) -> Result { +// let input = self.replace(&plan.input)?; +// +// Ok(PhysicalPlan::Exchange(Exchange { +// plan_id: plan.plan_id, +// input: Box::new(input), +// kind: plan.kind.clone(), +// keys: plan.keys.clone(), +// ignore_exchange: plan.ignore_exchange, +// allow_adjust_parallelism: plan.allow_adjust_parallelism, +// })) +// } +// +// fn replace_exchange_source(&mut self, plan: &ExchangeSource) -> Result { +// Ok(PhysicalPlan::ExchangeSource(plan.clone())) +// } +// +// fn replace_exchange_sink(&mut self, plan: &ExchangeSink) -> Result { +// let input = self.replace(&plan.input)?; +// +// Ok(PhysicalPlan::ExchangeSink(ExchangeSink { +// // TODO(leiysky): we reuse the plan id of the Exchange node here, +// // should generate a new one. +// plan_id: plan.plan_id, +// +// input: Box::new(input), +// schema: plan.schema.clone(), +// kind: plan.kind.clone(), +// keys: plan.keys.clone(), +// destination_fragment_id: plan.destination_fragment_id, +// query_id: plan.query_id.clone(), +// ignore_exchange: plan.ignore_exchange, +// allow_adjust_parallelism: plan.allow_adjust_parallelism, +// })) +// } +// +// fn replace_union(&mut self, plan: &UnionAll) -> Result { +// let left = self.replace(&plan.left)?; +// let right = self.replace(&plan.right)?; +// Ok(PhysicalPlan::UnionAll(UnionAll { +// plan_id: plan.plan_id, +// left: Box::new(left), +// right: Box::new(right), +// left_outputs: plan.left_outputs.clone(), +// right_outputs: plan.right_outputs.clone(), +// schema: plan.schema.clone(), +// stat_info: plan.stat_info.clone(), +// cte_scan_names: plan.cte_scan_names.clone(), +// })) +// } +// +// fn replace_copy_into_table(&mut self, plan: &CopyIntoTable) -> Result { +// match &plan.source { +// CopyIntoTableSource::Stage(_) => { +// Ok(PhysicalPlan::CopyIntoTable(Box::new(plan.clone()))) +// } +// CopyIntoTableSource::Query(query_physical_plan) => { +// let input = self.replace(query_physical_plan)?; +// Ok(PhysicalPlan::CopyIntoTable(Box::new(CopyIntoTable { +// source: CopyIntoTableSource::Query(Box::new(input)), +// ..plan.clone() +// }))) +// } +// } +// } +// +// fn replace_copy_into_location(&mut self, plan: &CopyIntoLocation) -> Result { +// let input = self.replace(&plan.input)?; +// +// Ok(PhysicalPlan::CopyIntoLocation(Box::new(CopyIntoLocation { +// plan_id: plan.plan_id, +// input: Box::new(input), +// project_columns: plan.project_columns.clone(), +// input_data_schema: plan.input_data_schema.clone(), +// input_table_schema: plan.input_table_schema.clone(), +// info: plan.info.clone(), +// }))) +// } +// +// fn replace_insert_select(&mut self, plan: &DistributedInsertSelect) -> Result { +// let input = self.replace(&plan.input)?; +// +// Ok(PhysicalPlan::DistributedInsertSelect(Box::new( +// DistributedInsertSelect { +// input: Box::new(input), +// ..plan.clone() +// }, +// ))) +// } +// +// fn replace_compact_source(&mut self, plan: &CompactSource) -> Result { +// Ok(PhysicalPlan::CompactSource(Box::new(plan.clone()))) +// } +// +// fn replace_commit_sink(&mut self, plan: &CommitSink) -> Result { +// let input = self.replace(&plan.input)?; +// Ok(PhysicalPlan::CommitSink(Box::new(CommitSink { +// input: Box::new(input), +// ..plan.clone() +// }))) +// } +// +// fn replace_async_sourcer(&mut self, plan: &ReplaceAsyncSourcer) -> Result { +// Ok(PhysicalPlan::ReplaceAsyncSourcer(plan.clone())) +// } +// +// fn replace_deduplicate(&mut self, plan: &ReplaceDeduplicate) -> Result { +// let input = self.replace(&plan.input)?; +// Ok(PhysicalPlan::ReplaceDeduplicate(Box::new( +// ReplaceDeduplicate { +// input: Box::new(input), +// ..plan.clone() +// }, +// ))) +// } +// +// fn replace_replace_into(&mut self, plan: &ReplaceInto) -> Result { +// let input = self.replace(&plan.input)?; +// Ok(PhysicalPlan::ReplaceInto(Box::new(ReplaceInto { +// input: Box::new(input), +// ..plan.clone() +// }))) +// } +// +// fn replace_mutation_source(&mut self, plan: &MutationSource) -> Result { +// Ok(PhysicalPlan::MutationSource(plan.clone())) +// } +// +// fn replace_column_mutation(&mut self, plan: &ColumnMutation) -> Result { +// let input = self.replace(&plan.input)?; +// Ok(PhysicalPlan::ColumnMutation(ColumnMutation { +// input: Box::new(input), +// ..plan.clone() +// })) +// } +// +// fn replace_mutation(&mut self, plan: &Mutation) -> Result { +// let input = self.replace(&plan.input)?; +// Ok(PhysicalPlan::Mutation(Box::new(Mutation { +// input: Box::new(input), +// ..plan.clone() +// }))) +// } +// +// fn replace_mutation_split(&mut self, plan: &MutationSplit) -> Result { +// let input = self.replace(&plan.input)?; +// Ok(PhysicalPlan::MutationSplit(Box::new(MutationSplit { +// input: Box::new(input), +// ..plan.clone() +// }))) +// } +// +// fn replace_mutation_manipulate(&mut self, plan: &MutationManipulate) -> Result { +// let input = self.replace(&plan.input)?; +// Ok(PhysicalPlan::MutationManipulate(Box::new( +// MutationManipulate { +// input: Box::new(input), +// ..plan.clone() +// }, +// ))) +// } +// +// fn replace_mutation_organize(&mut self, plan: &MutationOrganize) -> Result { +// let input = self.replace(&plan.input)?; +// Ok(PhysicalPlan::MutationOrganize(Box::new(MutationOrganize { +// input: Box::new(input), +// ..plan.clone() +// }))) +// } +// +// fn replace_add_stream_column(&mut self, plan: &AddStreamColumn) -> Result { +// let input = self.replace(&plan.input)?; +// Ok(PhysicalPlan::AddStreamColumn(Box::new(AddStreamColumn { +// input: Box::new(input), +// ..plan.clone() +// }))) +// } +// +// fn replace_project_set(&mut self, plan: &ProjectSet) -> Result { +// let input = self.replace(&plan.input)?; +// Ok(PhysicalPlan::ProjectSet(ProjectSet { +// plan_id: plan.plan_id, +// input: Box::new(input), +// srf_exprs: plan.srf_exprs.clone(), +// projections: plan.projections.clone(), +// stat_info: plan.stat_info.clone(), +// })) +// } +// +// fn replace_udf(&mut self, plan: &Udf) -> Result { +// let input = self.replace(&plan.input)?; +// Ok(PhysicalPlan::Udf(Udf { +// plan_id: plan.plan_id, +// input: Box::new(input), +// udf_funcs: plan.udf_funcs.clone(), +// stat_info: plan.stat_info.clone(), +// script_udf: plan.script_udf, +// })) +// } +// +// fn replace_async_function(&mut self, plan: &AsyncFunction) -> Result { +// let input = self.replace(&plan.input)?; +// Ok(PhysicalPlan::AsyncFunction(AsyncFunction { +// plan_id: plan.plan_id, +// input: Box::new(input), +// async_func_descs: plan.async_func_descs.clone(), +// stat_info: plan.stat_info.clone(), +// })) +// } +// +// fn replace_duplicate(&mut self, plan: &Duplicate) -> Result { +// let input = self.replace(&plan.input)?; +// Ok(PhysicalPlan::Duplicate(Box::new(Duplicate { +// input: Box::new(input), +// ..plan.clone() +// }))) +// } +// +// fn replace_shuffle(&mut self, plan: &Shuffle) -> Result { +// let input = self.replace(&plan.input)?; +// Ok(PhysicalPlan::Shuffle(Box::new(Shuffle { +// input: Box::new(input), +// ..plan.clone() +// }))) +// } +// +// fn replace_chunk_filter(&mut self, plan: &ChunkFilter) -> Result { +// let input = self.replace(&plan.input)?; +// Ok(PhysicalPlan::ChunkFilter(Box::new(ChunkFilter { +// input: Box::new(input), +// predicates: plan.predicates.clone(), +// ..plan.clone() +// }))) +// } +// +// fn replace_chunk_eval_scalar(&mut self, plan: &ChunkEvalScalar) -> Result { +// let input = self.replace(&plan.input)?; +// Ok(PhysicalPlan::ChunkEvalScalar(Box::new(ChunkEvalScalar { +// input: Box::new(input), +// ..plan.clone() +// }))) +// } +// +// fn replace_chunk_cast_schema(&mut self, plan: &ChunkCastSchema) -> Result { +// let input = self.replace(&plan.input)?; +// Ok(PhysicalPlan::ChunkCastSchema(Box::new(ChunkCastSchema { +// input: Box::new(input), +// cast_schemas: plan.cast_schemas.clone(), +// ..plan.clone() +// }))) +// } +// +// fn replace_chunk_fill_and_reorder( +// &mut self, +// plan: &ChunkFillAndReorder, +// ) -> Result { +// let input = self.replace(&plan.input)?; +// Ok(PhysicalPlan::ChunkFillAndReorder(Box::new( +// ChunkFillAndReorder { +// input: Box::new(input), +// ..plan.clone() +// }, +// ))) +// } +// +// fn replace_chunk_append_data(&mut self, plan: &ChunkAppendData) -> Result { +// let input = self.replace(&plan.input)?; +// Ok(PhysicalPlan::ChunkAppendData(Box::new(ChunkAppendData { +// input: Box::new(input), +// ..plan.clone() +// }))) +// } +// +// fn replace_chunk_merge(&mut self, plan: &ChunkMerge) -> Result { +// let input = self.replace(&plan.input)?; +// Ok(PhysicalPlan::ChunkMerge(Box::new(ChunkMerge { +// input: Box::new(input), +// ..plan.clone() +// }))) +// } +// +// fn replace_chunk_commit_insert(&mut self, plan: &ChunkCommitInsert) -> Result { +// let input = self.replace(&plan.input)?; +// Ok(PhysicalPlan::ChunkCommitInsert(Box::new( +// ChunkCommitInsert { +// input: Box::new(input), +// ..plan.clone() +// }, +// ))) +// } +// } impl PhysicalPlan { pub fn traverse<'a, 'b>( diff --git a/src/query/sql/src/executor/physical_plans/physical_add_stream_column.rs b/src/query/sql/src/executor/physical_plans/physical_add_stream_column.rs index 77d9ab7cb1630..3aadcdb7295ba 100644 --- a/src/query/sql/src/executor/physical_plans/physical_add_stream_column.rs +++ b/src/query/sql/src/executor/physical_plans/physical_add_stream_column.rs @@ -31,6 +31,7 @@ use crate::plans::ConstantExpr; use crate::plans::FunctionCall; use crate::Binder; use crate::ColumnBindingBuilder; +use crate::executor::physical_plan::{DeriveExt, PhysicalPlanDeriveHandle}; use crate::MetadataRef; use crate::ScalarExpr; use crate::Visibility; @@ -58,9 +59,23 @@ impl IPhysicalPlan for AddStreamColumn { Box::new(std::iter::once(&self.input)) } - fn children_mut<'a>(&'a mut self) -> Box> + 'a> { + fn children_mut<'a>(&'a mut self) -> Box> + 'a> { Box::new(std::iter::once(&mut self.input)) } + + fn derive_with(&self, handle: &mut Box) -> Box { + let derive_input = self.input.derive_with(handle); + + match handle.derive(self, vec![derive_input]) { + Ok(v) => v, + Err(children) => { + let mut new_add_stream_column = self.clone(); + assert_eq!(children.len(), 1); + new_add_stream_column.input = children[0]; + Box::new(new_add_stream_column) + } + } + } } impl AddStreamColumn { diff --git a/src/query/sql/src/executor/physical_plans/physical_aggregate_expand.rs b/src/query/sql/src/executor/physical_plans/physical_aggregate_expand.rs index fd676c62d3fdd..eda2e047e4853 100644 --- a/src/query/sql/src/executor/physical_plans/physical_aggregate_expand.rs +++ b/src/query/sql/src/executor/physical_plans/physical_aggregate_expand.rs @@ -22,6 +22,7 @@ use databend_common_expression::DataSchemaRefExt; use crate::executor::explain::PlanStatsInfo; use crate::executor::{IPhysicalPlan, PhysicalPlan, PhysicalPlanMeta}; +use crate::executor::physical_plan::PhysicalPlanDeriveHandle; use crate::plans::GroupingSets; use crate::IndexType; @@ -97,4 +98,18 @@ impl IPhysicalPlan for AggregateExpand { .collect::>() .join(", ")) } + + fn derive_with(&self, handle: &mut Box) -> Box { + let derive_input = self.input.derive_with(handle); + + match handle.derive(self, vec![derive_input]) { + Ok(v) => v, + Err(children) => { + let mut new_aggregate_expand = self.clone(); + assert_eq!(children.len(), 1); + new_aggregate_expand.input = children[0]; + Box::new(new_aggregate_expand) + } + } + } } diff --git a/src/query/sql/src/executor/physical_plans/physical_aggregate_final.rs b/src/query/sql/src/executor/physical_plans/physical_aggregate_final.rs index de30eb1bfc642..0c94ce2a3c9e9 100644 --- a/src/query/sql/src/executor/physical_plans/physical_aggregate_final.rs +++ b/src/query/sql/src/executor/physical_plans/physical_aggregate_final.rs @@ -36,6 +36,7 @@ use crate::optimizer::ir::SExpr; use crate::plans::AggregateMode; use crate::plans::DummyTableScan; use crate::ColumnSet; +use crate::executor::physical_plan::PhysicalPlanDeriveHandle; use crate::IndexType; use crate::ScalarExpr; @@ -106,6 +107,20 @@ impl IPhysicalPlan for AggregateFinal { Ok(labels) } + + fn derive_with(&self, handle: &mut Box) -> Box { + let derive_input = self.input.derive_with(handle); + + match handle.derive(self, vec![derive_input]) { + Ok(v) => v, + Err(children) => { + let mut new_aggregate_final = self.clone(); + assert_eq!(children.len(), 1); + new_aggregate_final.input = children[0]; + Box::new(new_aggregate_final) + } + } + } } impl PhysicalPlanBuilder { diff --git a/src/query/sql/src/executor/physical_plans/physical_aggregate_partial.rs b/src/query/sql/src/executor/physical_plans/physical_aggregate_partial.rs index a2261f8f146a8..3b3e0763b81a7 100644 --- a/src/query/sql/src/executor/physical_plans/physical_aggregate_partial.rs +++ b/src/query/sql/src/executor/physical_plans/physical_aggregate_partial.rs @@ -27,6 +27,7 @@ use super::SortDesc; use crate::executor::explain::PlanStatsInfo; use crate::executor::physical_plans::common::AggregateFunctionDesc; use crate::executor::{IPhysicalPlan, PhysicalPlan, PhysicalPlanMeta}; +use crate::executor::physical_plan::PhysicalPlanDeriveHandle; use crate::IndexType; #[derive(serde::Serialize, serde::Deserialize, Clone, Debug)] @@ -103,4 +104,18 @@ impl IPhysicalPlan for AggregatePartial { Ok(labels) } + + fn derive_with(&self, handle: &mut Box) -> Box { + let derive_input = self.input.derive_with(handle); + + match handle.derive(self, vec![derive_input]) { + Ok(v) => v, + Err(children) => { + let mut new_aggregate_partial = self.clone(); + assert_eq!(children.len(), 1); + new_aggregate_partial.input = children[0]; + Box::new(new_aggregate_partial) + } + } + } } diff --git a/src/query/sql/src/executor/physical_plans/physical_async_func.rs b/src/query/sql/src/executor/physical_plans/physical_async_func.rs index b93cb5877ec5b..a2e2e8ce76b92 100644 --- a/src/query/sql/src/executor/physical_plans/physical_async_func.rs +++ b/src/query/sql/src/executor/physical_plans/physical_async_func.rs @@ -27,6 +27,7 @@ use crate::executor::PhysicalPlanBuilder; use crate::optimizer::ir::SExpr; use crate::plans::AsyncFunctionArgument; use crate::ColumnSet; +use crate::executor::physical_plan::PhysicalPlanDeriveHandle; use crate::IndexType; use crate::ScalarExpr; @@ -80,6 +81,20 @@ impl IPhysicalPlan for AsyncFunction { .map(|x| x.display_name.clone()) .join(", ")) } + + fn derive_with(&self, handle: &mut Box) -> Box { + let derive_input = self.input.derive_with(handle); + + match handle.derive(self, vec![derive_input]) { + Ok(v) => v, + Err(children) => { + let mut new_async_function = self.clone(); + assert_eq!(children.len(), 1); + new_async_function.input = children[0]; + Box::new(new_async_function) + } + } + } } #[derive(Clone, Debug, Eq, PartialEq, serde::Serialize, serde::Deserialize)] diff --git a/src/query/sql/src/executor/physical_plans/physical_broadcast.rs b/src/query/sql/src/executor/physical_plans/physical_broadcast.rs index d38a22cfeade5..2014b885aa3a7 100644 --- a/src/query/sql/src/executor/physical_plans/physical_broadcast.rs +++ b/src/query/sql/src/executor/physical_plans/physical_broadcast.rs @@ -18,6 +18,7 @@ use databend_common_expression::DataSchemaRef; use super::Exchange; use super::FragmentKind; use crate::executor::{IPhysicalPlan, PhysicalPlan, PhysicalPlanMeta}; +use crate::executor::physical_plan::PhysicalPlanDeriveHandle; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct BroadcastSource { @@ -34,6 +35,17 @@ impl IPhysicalPlan for BroadcastSource { fn get_meta_mut(&mut self) -> &mut PhysicalPlanMeta { &mut self.meta } + + fn derive_with(&self, handle: &mut Box) -> Box { + match handle.derive(self, vec![]) { + Ok(v) => v, + Err(children) => { + assert!(children.is_empty()); + Box::new(self.clone()) + } + } + } + } #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] @@ -64,6 +76,20 @@ impl IPhysicalPlan for BroadcastSink { fn children_mut<'a>(&'a mut self) -> Box> + 'a> { Box::new(std::iter::once(&mut self.input)) } + + fn derive_with(&self, handle: &mut Box) -> Box { + let derive_input = self.input.derive_with(handle); + + match handle.derive(self, vec![derive_input]) { + Ok(v) => v, + Err(children) => { + let mut new_broadcast_sink = self.clone(); + assert_eq!(children.len(), 1); + new_broadcast_sink.input = children[0]; + Box::new(new_broadcast_sink) + } + } + } } pub fn build_broadcast_plan(broadcast_id: u32) -> Result> { diff --git a/src/query/sql/src/executor/physical_plans/physical_cache_scan.rs b/src/query/sql/src/executor/physical_plans/physical_cache_scan.rs index b112decffdf76..4cfe110d8895e 100644 --- a/src/query/sql/src/executor/physical_plans/physical_cache_scan.rs +++ b/src/query/sql/src/executor/physical_plans/physical_cache_scan.rs @@ -20,6 +20,7 @@ use crate::executor::{IPhysicalPlan, PhysicalPlan, PhysicalPlanMeta}; use crate::executor::PhysicalPlanBuilder; use crate::plans::CacheSource; use crate::ColumnSet; +use crate::executor::physical_plan::PhysicalPlanDeriveHandle; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct CacheScan { @@ -41,6 +42,16 @@ impl IPhysicalPlan for CacheScan { fn output_schema(&self) -> Result { Ok(self.output_schema.clone()) } + + fn derive_with(&self, handle: &mut Box) -> Box { + match handle.derive(self, vec![]) { + Ok(v) => v, + Err(children) => { + assert!(children.is_empty()); + Box::new(self.clone()) + } + } + } } impl PhysicalPlanBuilder { diff --git a/src/query/sql/src/executor/physical_plans/physical_column_mutation.rs b/src/query/sql/src/executor/physical_plans/physical_column_mutation.rs index 1d5bb8b11adda..f01427a43c663 100644 --- a/src/query/sql/src/executor/physical_plans/physical_column_mutation.rs +++ b/src/query/sql/src/executor/physical_plans/physical_column_mutation.rs @@ -18,7 +18,7 @@ use databend_common_expression::{DataSchemaRef, RemoteExpr}; use databend_common_meta_app::schema::TableInfo; use databend_storages_common_table_meta::meta::TableMetaTimestamps; use crate::executor::{IPhysicalPlan, PhysicalPlanMeta}; -use crate::executor::physical_plan::PhysicalPlan; +use crate::executor::physical_plan::{PhysicalPlanDeriveHandle, PhysicalPlan}; use crate::executor::physical_plans::MutationKind; use databend_common_exception::Result; @@ -57,4 +57,18 @@ impl IPhysicalPlan for ColumnMutation { fn children_mut<'a>(&'a mut self) -> Box> + 'a> { Box::new(std::iter::once(&mut self.input)) } + + fn derive_with(&self, handle: &mut Box) -> Box { + let derive_input = self.input.derive_with(handle); + + match handle.derive(self, vec![derive_input]) { + Ok(v) => v, + Err(children) => { + let mut new_column_mutation = self.clone(); + assert_eq!(children.len(), 1); + new_column_mutation.input = children[0]; + Box::new(new_column_mutation) + } + } + } } diff --git a/src/query/sql/src/executor/physical_plans/physical_commit_sink.rs b/src/query/sql/src/executor/physical_plans/physical_commit_sink.rs index bda53238a9565..b936a7120e8ae 100644 --- a/src/query/sql/src/executor/physical_plans/physical_commit_sink.rs +++ b/src/query/sql/src/executor/physical_plans/physical_commit_sink.rs @@ -25,6 +25,7 @@ use crate::executor::physical_plans::common::MutationKind; use crate::executor::{IPhysicalPlan, PhysicalPlan, PhysicalPlanMeta}; use crate::plans::TruncateMode; use databend_common_exception::Result; +use crate::executor::physical_plan::PhysicalPlanDeriveHandle; // serde is required by `PhysicalPlan` /// The commit sink is used to commit the data to the table. @@ -64,6 +65,20 @@ impl IPhysicalPlan for CommitSink { fn children_mut<'a>(&'a mut self) -> Box> + 'a> { Box::new(std::iter::once(&mut self.input)) } + + fn derive_with(&self, handle: &mut Box) -> Box { + let derive_input = self.input.derive_with(handle); + + match handle.derive(self, vec![derive_input]) { + Ok(v) => v, + Err(children) => { + let mut new_commit_sink = self.clone(); + assert_eq!(children.len(), 1); + new_commit_sink.input = children[0]; + Box::new(new_commit_sink) + } + } + } } #[derive(serde::Serialize, serde::Deserialize, Clone, Debug)] diff --git a/src/query/sql/src/executor/physical_plans/physical_compact_source.rs b/src/query/sql/src/executor/physical_plans/physical_compact_source.rs index 909ff175dcb5b..31900011978e5 100644 --- a/src/query/sql/src/executor/physical_plans/physical_compact_source.rs +++ b/src/query/sql/src/executor/physical_plans/physical_compact_source.rs @@ -29,6 +29,7 @@ use crate::executor::physical_plans::Exchange; use crate::executor::physical_plans::FragmentKind; use crate::executor::physical_plans::MutationKind; use crate::executor::{IPhysicalPlan, PhysicalPlan, PhysicalPlanMeta}; +use crate::executor::physical_plan::PhysicalPlanDeriveHandle; use crate::executor::PhysicalPlanBuilder; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] @@ -49,6 +50,16 @@ impl IPhysicalPlan for CompactSource { fn get_meta_mut(&mut self) -> &mut PhysicalPlanMeta { &mut self.meta } + + fn derive_with(&self, handle: &mut Box) -> Box { + match handle.derive(self, vec![]) { + Ok(v) => v, + Err(children) => { + assert!(children.is_empty()); + Box::new(self.clone()) + } + } + } } impl PhysicalPlanBuilder { diff --git a/src/query/sql/src/executor/physical_plans/physical_constant_table_scan.rs b/src/query/sql/src/executor/physical_plans/physical_constant_table_scan.rs index db74487e20cc8..c411512ed8c32 100644 --- a/src/query/sql/src/executor/physical_plans/physical_constant_table_scan.rs +++ b/src/query/sql/src/executor/physical_plans/physical_constant_table_scan.rs @@ -19,6 +19,7 @@ use databend_common_expression::DataSchemaRef; use crate::executor::{IPhysicalPlan, PhysicalPlan, PhysicalPlanMeta}; use crate::executor::PhysicalPlanBuilder; use crate::ColumnSet; +use crate::executor::physical_plan::PhysicalPlanDeriveHandle; use crate::IndexType; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] @@ -42,6 +43,16 @@ impl IPhysicalPlan for ConstantTableScan { fn output_schema(&self) -> Result { Ok(self.output_schema.clone()) } + + fn derive_with(&self, handle: &mut Box) -> Box { + match handle.derive(self, vec![]) { + Ok(v) => v, + Err(children) => { + assert!(children.is_empty()); + Box::new(self.clone()) + } + } + } } impl ConstantTableScan { diff --git a/src/query/sql/src/executor/physical_plans/physical_copy_into_location.rs b/src/query/sql/src/executor/physical_plans/physical_copy_into_location.rs index 4eff317c1283e..387bb848a4d54 100644 --- a/src/query/sql/src/executor/physical_plans/physical_copy_into_location.rs +++ b/src/query/sql/src/executor/physical_plans/physical_copy_into_location.rs @@ -24,6 +24,7 @@ use databend_storages_common_stage::CopyIntoLocationInfo; use crate::executor::{IPhysicalPlan, PhysicalPlan, PhysicalPlanMeta}; use crate::ColumnBinding; +use crate::executor::physical_plan::PhysicalPlanDeriveHandle; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct CopyIntoLocation { @@ -66,4 +67,18 @@ impl IPhysicalPlan for CopyIntoLocation { self.input.try_find_single_data_source() } + fn derive_with(&self, handle: &mut Box) -> Box { + let derive_input = self.input.derive_with(handle); + + match handle.derive(self, vec![derive_input]) { + Ok(v) => v, + Err(children) => { + let mut new_copy_into_location = self.clone(); + assert_eq!(children.len(), 1); + new_copy_into_location.input = children[0]; + Box::new(new_copy_into_location) + } + } + } + } diff --git a/src/query/sql/src/executor/physical_plans/physical_copy_into_table.rs b/src/query/sql/src/executor/physical_plans/physical_copy_into_table.rs index 14737fed539b2..9d052690d3b6e 100644 --- a/src/query/sql/src/executor/physical_plans/physical_copy_into_table.rs +++ b/src/query/sql/src/executor/physical_plans/physical_copy_into_table.rs @@ -21,7 +21,7 @@ use databend_common_meta_app::schema::TableInfo; use databend_storages_common_table_meta::meta::TableMetaTimestamps; use enum_as_inner::EnumAsInner; -use crate::executor::physical_plan::PhysicalPlan; +use crate::executor::physical_plan::{PhysicalPlanDeriveHandle, PhysicalPlan}; use crate::plans::CopyIntoTableMode; use crate::plans::ValidationMode; use crate::ColumnBinding; @@ -71,6 +71,20 @@ impl IPhysicalPlan for CopyIntoTable { CopyIntoTableSource::Stage(v) => Box::new(std::iter::once(v)), } } + + fn derive_with(&self, handle: &mut Box) -> Box { + let derive_input = self.input.derive_with(handle); + + match handle.derive(self, vec![derive_input]) { + Ok(v) => v, + Err(children) => { + let mut new_copy_into_table = self.clone(); + assert_eq!(children.len(), 1); + new_copy_into_table.input = children[0]; + Box::new(new_copy_into_table) + } + } + } } #[derive(Clone, Debug, serde::Serialize, serde::Deserialize, EnumAsInner)] diff --git a/src/query/sql/src/executor/physical_plans/physical_distributed_insert_select.rs b/src/query/sql/src/executor/physical_plans/physical_distributed_insert_select.rs index e9958d080b5ef..85cc2ef1ed378 100644 --- a/src/query/sql/src/executor/physical_plans/physical_distributed_insert_select.rs +++ b/src/query/sql/src/executor/physical_plans/physical_distributed_insert_select.rs @@ -20,6 +20,7 @@ use databend_storages_common_table_meta::meta::TableMetaTimestamps; use crate::executor::{IPhysicalPlan, PhysicalPlan, PhysicalPlanMeta}; use crate::ColumnBinding; use databend_common_exception::Result; +use crate::executor::physical_plan::PhysicalPlanDeriveHandle; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct DistributedInsertSelect { @@ -59,4 +60,18 @@ impl IPhysicalPlan for DistributedInsertSelect { fn try_find_single_data_source(&self) -> Option<&DataSourcePlan> { self.input.try_find_single_data_source() } + + fn derive_with(&self, handle: &mut Box) -> Box { + let derive_input = self.input.derive_with(handle); + + match handle.derive(self, vec![derive_input]) { + Ok(v) => v, + Err(children) => { + let mut new_distributed_insert_select = self.clone(); + assert_eq!(children.len(), 1); + new_distributed_insert_select.input = children[0]; + Box::new(new_distributed_insert_select) + } + } + } } diff --git a/src/query/sql/src/executor/physical_plans/physical_eval_scalar.rs b/src/query/sql/src/executor/physical_plans/physical_eval_scalar.rs index 6e1c7942339cc..24e00eae6f520 100644 --- a/src/query/sql/src/executor/physical_plans/physical_eval_scalar.rs +++ b/src/query/sql/src/executor/physical_plans/physical_eval_scalar.rs @@ -27,7 +27,7 @@ use databend_common_expression::Scalar; use databend_common_functions::BUILTIN_FUNCTIONS; use crate::executor::explain::PlanStatsInfo; -use crate::executor::physical_plan::PhysicalPlan; +use crate::executor::physical_plan::{PhysicalPlanDeriveHandle, PhysicalPlan}; use crate::executor::physical_plan_builder::PhysicalPlanBuilder; use crate::optimizer::ir::Matcher; use crate::optimizer::ir::SExpr; @@ -122,6 +122,20 @@ impl IPhysicalPlan for EvalScalar { ) ])) } + + fn derive_with(&self, handle: &mut Box) -> Box { + let derive_input = self.input.derive_with(handle); + + match handle.derive(self, vec![derive_input]) { + Ok(v) => v, + Err(children) => { + let mut new_eval_scalar = self.clone(); + assert_eq!(children.len(), 1); + new_eval_scalar.input = children[0]; + Box::new(new_eval_scalar) + } + } + } } impl PhysicalPlanBuilder { diff --git a/src/query/sql/src/executor/physical_plans/physical_exchange.rs b/src/query/sql/src/executor/physical_plans/physical_exchange.rs index da5bbba5ccd7a..787ca2ec2709e 100644 --- a/src/query/sql/src/executor/physical_plans/physical_exchange.rs +++ b/src/query/sql/src/executor/physical_plans/physical_exchange.rs @@ -24,6 +24,7 @@ use crate::executor::{IPhysicalPlan, PhysicalPlan, PhysicalPlanMeta}; use crate::executor::PhysicalPlanBuilder; use crate::optimizer::ir::SExpr; use crate::ColumnSet; +use crate::executor::physical_plan::PhysicalPlanDeriveHandle; use crate::TypeCheck; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] @@ -62,6 +63,20 @@ impl IPhysicalPlan for Exchange { fn is_distributed_plan(&self) -> bool { true } + + fn derive_with(&self, handle: &mut Box) -> Box { + let derive_input = self.input.derive_with(handle); + + match handle.derive(self, vec![derive_input]) { + Ok(v) => v, + Err(children) => { + let mut new_exchange = self.clone(); + assert_eq!(children.len(), 1); + new_exchange.input = children[0]; + Box::new(new_exchange) + } + } + } } impl PhysicalPlanBuilder { diff --git a/src/query/sql/src/executor/physical_plans/physical_exchange_sink.rs b/src/query/sql/src/executor/physical_plans/physical_exchange_sink.rs index f09fea69b7d5f..afcbfdf9414ce 100644 --- a/src/query/sql/src/executor/physical_plans/physical_exchange_sink.rs +++ b/src/query/sql/src/executor/physical_plans/physical_exchange_sink.rs @@ -19,6 +19,7 @@ use databend_common_expression::RemoteExpr; use crate::executor::physical_plans::common::FragmentKind; use crate::executor::{IPhysicalPlan, PhysicalPlan, PhysicalPlanMeta}; +use crate::executor::physical_plan::PhysicalPlanDeriveHandle; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct ExchangeSink { @@ -68,4 +69,18 @@ impl IPhysicalPlan for ExchangeSink { fn is_distributed_plan(&self) -> bool { true } + + fn derive_with(&self, handle: &mut Box) -> Box { + let derive_input = self.input.derive_with(handle); + + match handle.derive(self, vec![derive_input]) { + Ok(v) => v, + Err(children) => { + let mut new_exchange_sink = self.clone(); + assert_eq!(children.len(), 1); + new_exchange_sink.input = children[0]; + Box::new(new_exchange_sink) + } + } + } } diff --git a/src/query/sql/src/executor/physical_plans/physical_exchange_source.rs b/src/query/sql/src/executor/physical_plans/physical_exchange_source.rs index 0f2c2fed4eb41..778cb3322c7f9 100644 --- a/src/query/sql/src/executor/physical_plans/physical_exchange_source.rs +++ b/src/query/sql/src/executor/physical_plans/physical_exchange_source.rs @@ -16,6 +16,7 @@ use databend_common_catalog::plan::DataSourcePlan; use databend_common_exception::Result; use databend_common_expression::DataSchemaRef; use crate::executor::{IPhysicalPlan, PhysicalPlanMeta}; +use crate::executor::physical_plan::PhysicalPlanDeriveHandle; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct ExchangeSource { @@ -46,4 +47,14 @@ impl IPhysicalPlan for ExchangeSource { fn is_distributed_plan(&self) -> bool { true } + + fn derive_with(&self, handle: &mut Box) -> Box { + match handle.derive(self, vec![]) { + Ok(v) => v, + Err(children) => { + assert!(children.is_empty()); + Box::new(self.clone()) + } + } + } } diff --git a/src/query/sql/src/executor/physical_plans/physical_expression_scan.rs b/src/query/sql/src/executor/physical_plans/physical_expression_scan.rs index ee83c8540c555..6c837c9facc6e 100644 --- a/src/query/sql/src/executor/physical_plans/physical_expression_scan.rs +++ b/src/query/sql/src/executor/physical_plans/physical_expression_scan.rs @@ -22,6 +22,7 @@ use crate::executor::{IPhysicalPlan, PhysicalPlan, PhysicalPlanMeta}; use crate::executor::PhysicalPlanBuilder; use crate::optimizer::ir::SExpr; use crate::ColumnSet; +use crate::executor::physical_plan::PhysicalPlanDeriveHandle; use crate::TypeCheck; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] @@ -53,6 +54,20 @@ impl IPhysicalPlan for ExpressionScan { fn children_mut<'a>(&'a mut self) -> Box> + 'a> { Box::new(std::iter::once(&mut self.input)) } + + fn derive_with(&self, handle: &mut Box) -> Box { + let derive_input = self.input.derive_with(handle); + + match handle.derive(self, vec![derive_input]) { + Ok(v) => v, + Err(children) => { + let mut new_expression_scan = self.clone(); + assert_eq!(children.len(), 1); + new_expression_scan.input = children[0]; + Box::new(new_expression_scan) + } + } + } } impl PhysicalPlanBuilder { diff --git a/src/query/sql/src/executor/physical_plans/physical_filter.rs b/src/query/sql/src/executor/physical_plans/physical_filter.rs index 96e3ab61b35d5..a001ffdeada4b 100644 --- a/src/query/sql/src/executor/physical_plans/physical_filter.rs +++ b/src/query/sql/src/executor/physical_plans/physical_filter.rs @@ -27,6 +27,7 @@ use crate::executor::PhysicalPlan; use crate::executor::PhysicalPlanBuilder; use crate::optimizer::ir::SExpr; use crate::ColumnSet; +use crate::executor::physical_plan::PhysicalPlanDeriveHandle; use crate::TypeCheck; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] @@ -93,6 +94,20 @@ impl IPhysicalPlan for Filter { ) ])) } + + fn derive_with(&self, handle: &mut Box) -> Box { + let derive_input = self.input.derive_with(handle); + + match handle.derive(self, vec![derive_input]) { + Ok(v) => v, + Err(children) => { + let mut new_filter = self.clone(); + assert_eq!(children.len(), 1); + new_filter.input = children[0]; + Box::new(new_filter) + } + } + } } impl PhysicalPlanBuilder { diff --git a/src/query/sql/src/executor/physical_plans/physical_hash_join.rs b/src/query/sql/src/executor/physical_plans/physical_hash_join.rs index ca9a5ee39fc88..a4698d5ad64b6 100644 --- a/src/query/sql/src/executor/physical_plans/physical_hash_join.rs +++ b/src/query/sql/src/executor/physical_plans/physical_hash_join.rs @@ -38,6 +38,7 @@ use crate::plans::Join; use crate::plans::JoinType; use crate::ColumnEntry; use crate::ColumnSet; +use crate::executor::physical_plan::PhysicalPlanDeriveHandle; use crate::IndexType; use crate::ScalarExpr; use crate::TypeCheck; @@ -188,6 +189,22 @@ impl IPhysicalPlan for HashJoin { Ok(labels) } + + fn derive_with(&self, handle: &mut Box) -> Box { + let derive_probe = self.probe.derive_with(handle); + let derive_build = self.build.derive_with(handle); + + match handle.derive(self, vec![derive_probe, derive_build]) { + Ok(v) => v, + Err(children) => { + let mut new_hash_join = self.clone(); + assert_eq!(children.len(), 2); + new_hash_join.probe = children[0]; + new_hash_join.build = children[1]; + Box::new(new_hash_join) + } + } + } } impl PhysicalPlanBuilder { diff --git a/src/query/sql/src/executor/physical_plans/physical_limit.rs b/src/query/sql/src/executor/physical_plans/physical_limit.rs index 69d81536eaa78..9bc375972725e 100644 --- a/src/query/sql/src/executor/physical_plans/physical_limit.rs +++ b/src/query/sql/src/executor/physical_plans/physical_limit.rs @@ -27,6 +27,7 @@ use crate::executor::PhysicalPlanBuilder; use crate::optimizer::ir::SExpr; use crate::ColumnEntry; use crate::ColumnSet; +use crate::executor::physical_plan::PhysicalPlanDeriveHandle; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct Limit { @@ -79,6 +80,20 @@ impl IPhysicalPlan for Limit { Ok(labels) } + + fn derive_with(&self, handle: &mut Box) -> Box { + let derive_input = self.input.derive_with(handle); + + match handle.derive(self, vec![derive_input]) { + Ok(v) => v, + Err(children) => { + let mut new_limit = self.clone(); + assert_eq!(children.len(), 1); + new_limit.input = children[0]; + Box::new(new_limit) + } + } + } } impl PhysicalPlanBuilder { diff --git a/src/query/sql/src/executor/physical_plans/physical_multi_table_insert.rs b/src/query/sql/src/executor/physical_plans/physical_multi_table_insert.rs index cd4ef71dfec15..dc90adf8c0e60 100644 --- a/src/query/sql/src/executor/physical_plans/physical_multi_table_insert.rs +++ b/src/query/sql/src/executor/physical_plans/physical_multi_table_insert.rs @@ -25,6 +25,8 @@ use databend_storages_common_table_meta::meta::TableMetaTimestamps; use crate::executor::{IPhysicalPlan, PhysicalPlan, PhysicalPlanMeta}; use crate::ColumnSet; +use crate::executor::physical_plan::PhysicalPlanDeriveHandle; + #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct Duplicate { pub meta: PhysicalPlanMeta, @@ -49,6 +51,20 @@ impl IPhysicalPlan for Duplicate { fn children_mut<'a>(&'a mut self) -> Box> + 'a> { Box::new(std::iter::once(&mut self.input)) } + + fn derive_with(&self, handle: &mut Box) -> Box { + let derive_input = self.input.derive_with(handle); + + match handle.derive(self, vec![derive_input]) { + Ok(v) => v, + Err(children) => { + let mut new_duplicate = self.clone(); + assert_eq!(children.len(), 1); + new_duplicate.input = children[0]; + Box::new(new_duplicate) + } + } + } } #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] @@ -75,6 +91,20 @@ impl IPhysicalPlan for Shuffle { fn children_mut<'a>(&'a mut self) -> Box> + 'a> { Box::new(std::iter::once(&mut self.input)) } + + fn derive_with(&self, handle: &mut Box) -> Box { + let derive_input = self.input.derive_with(handle); + + match handle.derive(self, vec![derive_input]) { + Ok(v) => v, + Err(children) => { + let mut new_shuffle = self.clone(); + assert_eq!(children.len(), 1); + new_shuffle.input = children[0]; + Box::new(new_shuffle) + } + } + } } #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] @@ -128,6 +158,20 @@ impl IPhysicalPlan for ChunkFilter { fn children_mut<'a>(&'a mut self) -> Box> + 'a> { Box::new(std::iter::once(&mut self.input)) } + + fn derive_with(&self, handle: &mut Box) -> Box { + let derive_input = self.input.derive_with(handle); + + match handle.derive(self, vec![derive_input]) { + Ok(v) => v, + Err(children) => { + let mut new_chunk_filter = self.clone(); + assert_eq!(children.len(), 1); + new_chunk_filter.input = children[0]; + Box::new(new_chunk_filter) + } + } + } } #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] @@ -154,6 +198,20 @@ impl IPhysicalPlan for ChunkEvalScalar { fn children_mut<'a>(&'a mut self) -> Box> + 'a> { Box::new(std::iter::once(&mut self.input)) } + + fn derive_with(&self, handle: &mut Box) -> Box { + let derive_input = self.input.derive_with(handle); + + match handle.derive(self, vec![derive_input]) { + Ok(v) => v, + Err(children) => { + let mut new_chunk_eval_scalar = self.clone(); + assert_eq!(children.len(), 1); + new_chunk_eval_scalar.input = children[0]; + Box::new(new_chunk_eval_scalar) + } + } + } } #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] @@ -186,6 +244,20 @@ impl IPhysicalPlan for ChunkCastSchema { fn children_mut<'a>(&'a mut self) -> Box> + 'a> { Box::new(std::iter::once(&mut self.input)) } + + fn derive_with(&self, handle: &mut Box) -> Box { + let derive_input = self.input.derive_with(handle); + + match handle.derive(self, vec![derive_input]) { + Ok(v) => v, + Err(children) => { + let mut new_chunk_cast_schema = self.clone(); + assert_eq!(children.len(), 1); + new_chunk_cast_schema.input = children[0]; + Box::new(new_chunk_cast_schema) + } + } + } } #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] @@ -218,6 +290,20 @@ impl IPhysicalPlan for ChunkFillAndReorder { fn children_mut<'a>(&'a mut self) -> Box> + 'a> { Box::new(std::iter::once(&mut self.input)) } + + fn derive_with(&self, handle: &mut Box) -> Box { + let derive_input = self.input.derive_with(handle); + + match handle.derive(self, vec![derive_input]) { + Ok(v) => v, + Err(children) => { + let mut new_chunk_fill_and_reorder = self.clone(); + assert_eq!(children.len(), 1); + new_chunk_fill_and_reorder.input = children[0]; + Box::new(new_chunk_fill_and_reorder) + } + } + } } #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] @@ -250,6 +336,20 @@ impl IPhysicalPlan for ChunkAppendData { fn children_mut<'a>(&'a mut self) -> Box> + 'a> { Box::new(std::iter::once(&mut self.input)) } + + fn derive_with(&self, handle: &mut Box) -> Box { + let derive_input = self.input.derive_with(handle); + + match handle.derive(self, vec![derive_input]) { + Ok(v) => v, + Err(children) => { + let mut new_chunk_append_data = self.clone(); + assert_eq!(children.len(), 1); + new_chunk_append_data.input = children[0]; + Box::new(new_chunk_append_data) + } + } + } } #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] @@ -283,6 +383,20 @@ impl IPhysicalPlan for ChunkMerge { fn children_mut<'a>(&'a mut self) -> Box> + 'a> { Box::new(std::iter::once(&mut self.input)) } + + fn derive_with(&self, handle: &mut Box) -> Box { + let derive_input = self.input.derive_with(handle); + + match handle.derive(self, vec![derive_input]) { + Ok(v) => v, + Err(children) => { + let mut new_chunk_merge = self.clone(); + assert_eq!(children.len(), 1); + new_chunk_merge.input = children[0]; + Box::new(new_chunk_merge) + } + } + } } #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] @@ -312,4 +426,18 @@ impl IPhysicalPlan for ChunkCommitInsert { fn children_mut<'a>(&'a mut self) -> Box> + 'a> { Box::new(std::iter::once(&mut self.input)) } + + fn derive_with(&self, handle: &mut Box) -> Box { + let derive_input = self.input.derive_with(handle); + + match handle.derive(self, vec![derive_input]) { + Ok(v) => v, + Err(children) => { + let mut new_chunk_commit_insert = self.clone(); + assert_eq!(children.len(), 1); + new_chunk_commit_insert.input = children[0]; + Box::new(new_chunk_commit_insert) + } + } + } } diff --git a/src/query/sql/src/executor/physical_plans/physical_mutation.rs b/src/query/sql/src/executor/physical_plans/physical_mutation.rs index fa7ff06ea928d..e3b865159e352 100644 --- a/src/query/sql/src/executor/physical_plans/physical_mutation.rs +++ b/src/query/sql/src/executor/physical_plans/physical_mutation.rs @@ -48,7 +48,7 @@ use super::CommitType; use crate::binder::wrap_cast; use crate::binder::MutationStrategy; use crate::binder::MutationType; -use crate::executor::physical_plan::PhysicalPlan; +use crate::executor::physical_plan::{PhysicalPlanDeriveHandle, PhysicalPlan}; use crate::executor::physical_plans::CommitSink; use crate::executor::physical_plans::Exchange; use crate::executor::physical_plans::FragmentKind; @@ -116,6 +116,20 @@ impl IPhysicalPlan for Mutation { fn children_mut<'a>(&'a mut self) -> Box> + 'a> { Box::new(std::iter::once(&mut self.input)) } + + fn derive_with(&self, handle: &mut Box) -> Box { + let derive_input = self.input.derive_with(handle); + + match handle.derive(self, vec![derive_input]) { + Ok(v) => v, + Err(children) => { + let mut new_mutation = self.clone(); + assert_eq!(children.len(), 1); + new_mutation.input = children[0]; + Box::new(new_mutation) + } + } + } } impl PhysicalPlanBuilder { diff --git a/src/query/sql/src/executor/physical_plans/physical_mutation_into_organize.rs b/src/query/sql/src/executor/physical_plans/physical_mutation_into_organize.rs index 2371b56c634ba..8feccff28c7a6 100644 --- a/src/query/sql/src/executor/physical_plans/physical_mutation_into_organize.rs +++ b/src/query/sql/src/executor/physical_plans/physical_mutation_into_organize.rs @@ -17,7 +17,7 @@ use databend_common_expression::DataSchemaRef; use crate::binder::MutationStrategy; use crate::executor::{IPhysicalPlan, PhysicalPlanMeta}; -use crate::executor::physical_plan::PhysicalPlan; +use crate::executor::physical_plan::{PhysicalPlanDeriveHandle, PhysicalPlan}; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct MutationOrganize { @@ -43,4 +43,18 @@ impl IPhysicalPlan for MutationOrganize { fn children_mut<'a>(&'a mut self) -> Box> + 'a> { Box::new(std::iter::once(&mut self.input)) } + + fn derive_with(&self, handle: &mut Box) -> Box { + let derive_input = self.input.derive_with(handle); + + match handle.derive(self, vec![derive_input]) { + Ok(v) => v, + Err(children) => { + let mut new_mutation_organize = self.clone(); + assert_eq!(children.len(), 1); + new_mutation_organize.input = children[0]; + Box::new(new_mutation_organize) + } + } + } } diff --git a/src/query/sql/src/executor/physical_plans/physical_mutation_into_split.rs b/src/query/sql/src/executor/physical_plans/physical_mutation_into_split.rs index dbc836bbf31d4..18bb39a19d412 100644 --- a/src/query/sql/src/executor/physical_plans/physical_mutation_into_split.rs +++ b/src/query/sql/src/executor/physical_plans/physical_mutation_into_split.rs @@ -15,7 +15,7 @@ use databend_common_exception::Result; use databend_common_expression::DataSchemaRef; use crate::executor::{IPhysicalPlan, PhysicalPlanMeta}; -use crate::executor::physical_plan::PhysicalPlan; +use crate::executor::physical_plan::{PhysicalPlanDeriveHandle, PhysicalPlan}; use crate::IndexType; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] @@ -42,4 +42,18 @@ impl IPhysicalPlan for MutationSplit { fn children_mut<'a>(&'a mut self) -> Box> + 'a> { Box::new(std::iter::once(&mut self.input)) } + + fn derive_with(&self, handle: &mut Box) -> Box { + let derive_input = self.input.derive_with(handle); + + match handle.derive(self, vec![derive_input]) { + Ok(v) => v, + Err(children) => { + let mut new_mutation_split = self.clone(); + assert_eq!(children.len(), 1); + new_mutation_split.input = children[0]; + Box::new(new_mutation_split) + } + } + } } diff --git a/src/query/sql/src/executor/physical_plans/physical_mutation_manipulate.rs b/src/query/sql/src/executor/physical_plans/physical_mutation_manipulate.rs index 23b8fe56d0a3f..225620af61cec 100644 --- a/src/query/sql/src/executor/physical_plans/physical_mutation_manipulate.rs +++ b/src/query/sql/src/executor/physical_plans/physical_mutation_manipulate.rs @@ -22,7 +22,7 @@ use databend_common_meta_app::schema::TableInfo; use crate::binder::MutationStrategy; use crate::executor::{IPhysicalPlan, PhysicalPlanMeta}; -use crate::executor::physical_plan::PhysicalPlan; +use crate::executor::physical_plan::{PhysicalPlanDeriveHandle, PhysicalPlan}; pub type MatchExpr = Vec<(Option, Option>)>; @@ -61,4 +61,18 @@ impl IPhysicalPlan for MutationManipulate { fn children_mut<'a>(&'a mut self) -> Box> + 'a> { Box::new(std::iter::once(&mut self.input)) } + + fn derive_with(&self, handle: &mut Box) -> Box { + let derive_input = self.input.derive_with(handle); + + match handle.derive(self, vec![derive_input]) { + Ok(v) => v, + Err(children) => { + let mut new_mutation_manipulate = self.clone(); + assert_eq!(children.len(), 1); + new_mutation_manipulate.input = children[0]; + Box::new(new_mutation_manipulate) + } + } + } } diff --git a/src/query/sql/src/executor/physical_plans/physical_mutation_source.rs b/src/query/sql/src/executor/physical_plans/physical_mutation_source.rs index f41513a60d5de..65c8827f3ea59 100644 --- a/src/query/sql/src/executor/physical_plans/physical_mutation_source.rs +++ b/src/query/sql/src/executor/physical_plans/physical_mutation_source.rs @@ -31,6 +31,7 @@ use crate::executor::{cast_expr_to_non_null_boolean, IPhysicalPlan, PhysicalPlan use crate::executor::PhysicalPlan; use crate::executor::PhysicalPlanBuilder; use crate::ColumnSet; +use crate::executor::physical_plan::PhysicalPlanDeriveHandle; use crate::IndexType; use crate::ScalarExpr; @@ -66,6 +67,16 @@ impl IPhysicalPlan for MutationSource { fn try_find_mutation_source(&self) -> Option { Some(self.clone()) } + + fn derive_with(&self, handle: &mut Box) -> Box { + match handle.derive(self, vec![]) { + Ok(v) => v, + Err(children) => { + assert!(children.is_empty()); + Box::new(self.clone()) + } + } + } } impl PhysicalPlanBuilder { diff --git a/src/query/sql/src/executor/physical_plans/physical_project_set.rs b/src/query/sql/src/executor/physical_plans/physical_project_set.rs index d58f5c7d252f8..ae1fcc48b6c73 100644 --- a/src/query/sql/src/executor/physical_plans/physical_project_set.rs +++ b/src/query/sql/src/executor/physical_plans/physical_project_set.rs @@ -27,6 +27,7 @@ use crate::executor::{IPhysicalPlan, PhysicalPlan, PhysicalPlanMeta}; use crate::executor::PhysicalPlanBuilder; use crate::optimizer::ir::SExpr; use crate::ColumnSet; +use crate::executor::physical_plan::PhysicalPlanDeriveHandle; use crate::IndexType; use crate::TypeCheck; @@ -88,6 +89,20 @@ impl IPhysicalPlan for ProjectSet { .map(|(x, _)| x.as_expr(&BUILTIN_FUNCTIONS).sql_display()) .join(", ")) } + + fn derive_with(&self, handle: &mut Box) -> Box { + let derive_input = self.input.derive_with(handle); + + match handle.derive(self, vec![derive_input]) { + Ok(v) => v, + Err(children) => { + let mut new_project_set = self.clone(); + assert_eq!(children.len(), 1); + new_project_set.input = children[0]; + Box::new(new_project_set) + } + } + } } impl PhysicalPlanBuilder { diff --git a/src/query/sql/src/executor/physical_plans/physical_r_cte_scan.rs b/src/query/sql/src/executor/physical_plans/physical_r_cte_scan.rs index b544c37402333..81d7d43ead0c3 100644 --- a/src/query/sql/src/executor/physical_plans/physical_r_cte_scan.rs +++ b/src/query/sql/src/executor/physical_plans/physical_r_cte_scan.rs @@ -20,6 +20,7 @@ use databend_common_expression::DataSchemaRefExt; use crate::executor::explain::PlanStatsInfo; use crate::executor::{IPhysicalPlan, PhysicalPlan, PhysicalPlanMeta}; +use crate::executor::physical_plan::PhysicalPlanDeriveHandle; use crate::executor::PhysicalPlanBuilder; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] @@ -43,6 +44,16 @@ impl IPhysicalPlan for RecursiveCteScan { fn output_schema(&self) -> Result { Ok(self.output_schema.clone()) } + + fn derive_with(&self, handle: &mut Box) -> Box { + match handle.derive(self, vec![]) { + Ok(v) => v, + Err(children) => { + assert!(children.is_empty()); + Box::new(self.clone()) + } + } + } } impl PhysicalPlanBuilder { diff --git a/src/query/sql/src/executor/physical_plans/physical_range_join.rs b/src/query/sql/src/executor/physical_plans/physical_range_join.rs index 95b3ef02d5b78..a0a6b21fb83c8 100644 --- a/src/query/sql/src/executor/physical_plans/physical_range_join.rs +++ b/src/query/sql/src/executor/physical_plans/physical_range_join.rs @@ -34,6 +34,7 @@ use crate::optimizer::ir::SExpr; use crate::plans::Join; use crate::plans::JoinType; use crate::ColumnSet; +use crate::executor::physical_plan::PhysicalPlanDeriveHandle; use crate::ScalarExpr; use crate::TypeCheck; @@ -103,6 +104,22 @@ impl IPhysicalPlan for RangeJoin { Ok(condition.join(" AND ")) } + + fn derive_with(&self, handle: &mut Box) -> Box { + let derive_left = self.left.derive_with(handle); + let derive_right = self.right.derive_with(handle); + + match handle.derive(self, vec![derive_left, derive_right]) { + Ok(v) => v, + Err(children) => { + let mut new_range_join = self.clone(); + assert_eq!(children.len(), 2); + new_range_join.left = children[0]; + new_range_join.right = children[1]; + Box::new(new_range_join) + } + } + } } #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] diff --git a/src/query/sql/src/executor/physical_plans/physical_recluster.rs b/src/query/sql/src/executor/physical_plans/physical_recluster.rs index 15caa374197ff..e15a00f937964 100644 --- a/src/query/sql/src/executor/physical_plans/physical_recluster.rs +++ b/src/query/sql/src/executor/physical_plans/physical_recluster.rs @@ -19,6 +19,7 @@ use databend_storages_common_table_meta::meta::TableMetaTimestamps; use crate::executor::{IPhysicalPlan, PhysicalPlan, PhysicalPlanMeta}; use databend_common_exception::Result; +use crate::executor::physical_plan::PhysicalPlanDeriveHandle; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct Recluster { @@ -37,6 +38,16 @@ impl IPhysicalPlan for Recluster { fn get_meta_mut(&mut self) -> &mut PhysicalPlanMeta { &mut self.meta } + + fn derive_with(&self, handle: &mut Box) -> Box { + match handle.derive(self, vec![]) { + Ok(v) => v, + Err(children) => { + assert!(children.is_empty()); + Box::new(self.clone()) + } + } + } } #[derive(serde::Serialize, serde::Deserialize, Clone, Debug)] @@ -70,4 +81,18 @@ impl IPhysicalPlan for HilbertPartition { fn children_mut<'a>(&'a mut self) -> Box> + 'a> { Box::new(std::iter::once(&mut self.input)) } + + fn derive_with(&self, handle: &mut Box) -> Box { + let derive_input = self.input.derive_with(handle); + + match handle.derive(self, vec![derive_input]) { + Ok(v) => v, + Err(children) => { + let mut new_hilbert_partition = self.clone(); + assert_eq!(children.len(), 1); + new_hilbert_partition.input = children[0]; + Box::new(new_hilbert_partition) + } + } + } } diff --git a/src/query/sql/src/executor/physical_plans/physical_replace_async_source.rs b/src/query/sql/src/executor/physical_plans/physical_replace_async_source.rs index 89c6f9b37d7a9..c35d86d91a836 100644 --- a/src/query/sql/src/executor/physical_plans/physical_replace_async_source.rs +++ b/src/query/sql/src/executor/physical_plans/physical_replace_async_source.rs @@ -15,6 +15,7 @@ use databend_common_catalog::plan::DataSourcePlan; use databend_common_expression::DataSchemaRef; use crate::executor::{IPhysicalPlan, PhysicalPlanMeta}; +use crate::executor::physical_plan::PhysicalPlanDeriveHandle; use crate::plans::InsertValue; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] @@ -33,4 +34,14 @@ impl IPhysicalPlan for ReplaceAsyncSourcer { fn get_meta_mut(&mut self) -> &mut PhysicalPlanMeta { &mut self.meta } + + fn derive_with(&self, handle: &mut Box) -> Box { + match handle.derive(self, vec![]) { + Ok(v) => v, + Err(children) => { + assert!(children.is_empty()); + Box::new(self.clone()) + } + } + } } diff --git a/src/query/sql/src/executor/physical_plans/physical_replace_deduplicate.rs b/src/query/sql/src/executor/physical_plans/physical_replace_deduplicate.rs index 2435988226d3e..7687a16da4bad 100644 --- a/src/query/sql/src/executor/physical_plans/physical_replace_deduplicate.rs +++ b/src/query/sql/src/executor/physical_plans/physical_replace_deduplicate.rs @@ -26,6 +26,7 @@ use crate::executor::physical_plans::common::OnConflictField; use crate::executor::{IPhysicalPlan, PhysicalPlan, PhysicalPlanMeta}; use crate::ColumnBinding; use databend_common_exception::Result; +use crate::executor::physical_plan::PhysicalPlanDeriveHandle; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct ReplaceDeduplicate { @@ -63,6 +64,20 @@ impl IPhysicalPlan for ReplaceDeduplicate { fn children_mut<'a>(&'a mut self) -> Box> + 'a> { Box::new(std::iter::once(&mut self.input)) } + + fn derive_with(&self, handle: &mut Box) -> Box { + let derive_input = self.input.derive_with(handle); + + match handle.derive(self, vec![derive_input]) { + Ok(v) => v, + Err(children) => { + let mut new_replace_deduplicate = self.clone(); + assert_eq!(children.len(), 1); + new_replace_deduplicate.input = children[0]; + Box::new(new_replace_deduplicate) + } + } + } } #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] diff --git a/src/query/sql/src/executor/physical_plans/physical_replace_into.rs b/src/query/sql/src/executor/physical_plans/physical_replace_into.rs index 079ff5ac5859f..891b8444727ea 100644 --- a/src/query/sql/src/executor/physical_plans/physical_replace_into.rs +++ b/src/query/sql/src/executor/physical_plans/physical_replace_into.rs @@ -23,6 +23,7 @@ use databend_storages_common_table_meta::meta::TableMetaTimestamps; use crate::executor::physical_plans::common::OnConflictField; use crate::executor::{IPhysicalPlan, PhysicalPlan, PhysicalPlanMeta}; use databend_common_exception::Result; +use crate::executor::physical_plan::PhysicalPlanDeriveHandle; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct ReplaceInto { @@ -59,4 +60,18 @@ impl IPhysicalPlan for ReplaceInto { fn children_mut<'a>(&'a mut self) -> Box> + 'a> { Box::new(std::iter::once(&mut self.input)) } + + fn derive_with(&self, handle: &mut Box) -> Box { + let derive_input = self.input.derive_with(handle); + + match handle.derive(self, vec![derive_input]) { + Ok(v) => v, + Err(children) => { + let mut new_replace_into = self.clone(); + assert_eq!(children.len(), 1); + new_replace_into.input = children[0]; + Box::new(new_replace_into) + } + } + } } diff --git a/src/query/sql/src/executor/physical_plans/physical_row_fetch.rs b/src/query/sql/src/executor/physical_plans/physical_row_fetch.rs index 003f4c87bce3a..96ae64e5dcc81 100644 --- a/src/query/sql/src/executor/physical_plans/physical_row_fetch.rs +++ b/src/query/sql/src/executor/physical_plans/physical_row_fetch.rs @@ -22,6 +22,7 @@ use databend_common_expression::DataSchemaRefExt; use crate::executor::explain::PlanStatsInfo; use crate::executor::{IPhysicalPlan, PhysicalPlan, PhysicalPlanMeta}; +use crate::executor::physical_plan::PhysicalPlanDeriveHandle; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct RowFetch { @@ -73,4 +74,18 @@ impl IPhysicalPlan for RowFetch { let projected_schema = self.cols_to_fetch.project_schema(&table_schema); Ok(projected_schema.fields.iter().map(|f| f.name()).join(", ")) } + + fn derive_with(&self, handle: &mut Box) -> Box { + let derive_input = self.input.derive_with(handle); + + match handle.derive(self, vec![derive_input]) { + Ok(v) => v, + Err(children) => { + let mut new_row_fetch = self.clone(); + assert_eq!(children.len(), 1); + new_row_fetch.input = children[0]; + Box::new(new_row_fetch) + } + } + } } diff --git a/src/query/sql/src/executor/physical_plans/physical_sort.rs b/src/query/sql/src/executor/physical_plans/physical_sort.rs index 72f85be8846ef..284bd2f88401c 100644 --- a/src/query/sql/src/executor/physical_plans/physical_sort.rs +++ b/src/query/sql/src/executor/physical_plans/physical_sort.rs @@ -32,6 +32,7 @@ use crate::executor::PhysicalPlanBuilder; use crate::optimizer::ir::SExpr; use crate::plans::WindowFuncType; use crate::ColumnSet; +use crate::executor::physical_plan::PhysicalPlanDeriveHandle; use crate::IndexType; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] @@ -126,6 +127,20 @@ impl IPhysicalPlan for Sort { .join(", ") ) } + + fn derive_with(&self, handle: &mut Box) -> Box { + let derive_input = self.input.derive_with(handle); + + match handle.derive(self, vec![derive_input]) { + Ok(v) => v, + Err(children) => { + let mut new_sort = self.clone(); + assert_eq!(children.len(), 1); + new_sort.input = children[0]; + Box::new(new_sort) + } + } + } } impl Sort { diff --git a/src/query/sql/src/executor/physical_plans/physical_table_scan.rs b/src/query/sql/src/executor/physical_plans/physical_table_scan.rs index eb19aa9c2a268..eaef4fed1daa2 100644 --- a/src/query/sql/src/executor/physical_plans/physical_table_scan.rs +++ b/src/query/sql/src/executor/physical_plans/physical_table_scan.rs @@ -70,6 +70,7 @@ use crate::TypeCheck; use crate::VirtualColumn; use crate::DUMMY_COLUMN_INDEX; use crate::DUMMY_TABLE_INDEX; +use crate::executor::physical_plan::PhysicalPlanDeriveHandle; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct TableScan { @@ -166,6 +167,16 @@ impl IPhysicalPlan for TableScan { .to_string()]) ])) } + + fn derive_with(&self, handle: &mut Box) -> Box { + match handle.derive(self, vec![]) { + Ok(v) => v, + Err(children) => { + assert!(children.is_empty()); + Box::new(self.clone()) + } + } + } } impl TableScan { diff --git a/src/query/sql/src/executor/physical_plans/physical_udf.rs b/src/query/sql/src/executor/physical_plans/physical_udf.rs index f51443021aa09..5c2f47cc061ff 100644 --- a/src/query/sql/src/executor/physical_plans/physical_udf.rs +++ b/src/query/sql/src/executor/physical_plans/physical_udf.rs @@ -29,6 +29,7 @@ use crate::executor::PhysicalPlanBuilder; use crate::optimizer::ir::SExpr; use crate::plans::UDFType; use crate::ColumnSet; +use crate::executor::physical_plan::PhysicalPlanDeriveHandle; use crate::IndexType; use crate::ScalarExpr; @@ -83,6 +84,20 @@ impl IPhysicalPlan for Udf { .map(|x| format!("{}({})", x.func_name, x.arg_exprs.join(", "))) .join(", ")) } + + fn derive_with(&self, handle: &mut Box) -> Box { + let derive_input = self.input.derive_with(handle); + + match handle.derive(self, vec![derive_input]) { + Ok(v) => v, + Err(children) => { + let mut new_udf = self.clone(); + assert_eq!(children.len(), 1); + new_udf.input = children[0]; + Box::new(new_udf) + } + } + } } #[derive(Clone, Debug, Eq, Hash, PartialEq, serde::Serialize, serde::Deserialize)] diff --git a/src/query/sql/src/executor/physical_plans/physical_union_all.rs b/src/query/sql/src/executor/physical_plans/physical_union_all.rs index 29ff2198645f9..31eebbf8e91d6 100644 --- a/src/query/sql/src/executor/physical_plans/physical_union_all.rs +++ b/src/query/sql/src/executor/physical_plans/physical_union_all.rs @@ -25,6 +25,7 @@ use crate::executor::{IPhysicalPlan, PhysicalPlan, PhysicalPlanMeta}; use crate::executor::PhysicalPlanBuilder; use crate::optimizer::ir::SExpr; use crate::ColumnSet; +use crate::executor::physical_plan::PhysicalPlanDeriveHandle; use crate::IndexType; use crate::ScalarExpr; use crate::TypeCheck; @@ -73,6 +74,22 @@ impl IPhysicalPlan for UnionAll { .map(|(l, r)| format!("#{} <- #{}", l.0, r.0)) .join(", ")) } + + fn derive_with(&self, handle: &mut Box) -> Box { + let derive_left = self.left.derive_with(handle); + let derive_right = self.right.derive_with(handle); + + match handle.derive(self, vec![derive_left, derive_right]) { + Ok(v) => v, + Err(children) => { + let mut new_union_all = self.clone(); + assert_eq!(children.len(), 2); + new_union_all.left = children[0]; + new_union_all.right = children[1]; + Box::new(new_union_all) + } + } + } } impl PhysicalPlanBuilder { diff --git a/src/query/sql/src/executor/physical_plans/physical_window.rs b/src/query/sql/src/executor/physical_plans/physical_window.rs index ded448a1b34c7..92136522cd19d 100644 --- a/src/query/sql/src/executor/physical_plans/physical_window.rs +++ b/src/query/sql/src/executor/physical_plans/physical_window.rs @@ -42,6 +42,7 @@ use crate::plans::WindowFuncFrame; use crate::plans::WindowFuncFrameBound; use crate::plans::WindowFuncType; use crate::ColumnSet; +use crate::executor::physical_plan::PhysicalPlanDeriveHandle; use crate::IndexType; use crate::ScalarExpr; use crate::TypeCheck; @@ -116,6 +117,20 @@ impl IPhysicalPlan for Window { Ok(format!("partition by {}, order by {}", partition_by, order_by)) } + + fn derive_with(&self, handle: &mut Box) -> Box { + let derive_input = self.input.derive_with(handle); + + match handle.derive(self, vec![derive_input]) { + Ok(v) => v, + Err(children) => { + let mut new_window = self.clone(); + assert_eq!(children.len(), 1); + new_window.input = children[0]; + Box::new(new_window) + } + } + } } #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] diff --git a/src/query/sql/src/executor/physical_plans/physical_window_partition.rs b/src/query/sql/src/executor/physical_plans/physical_window_partition.rs index f24282e4466b8..8eba8f7e69fa0 100644 --- a/src/query/sql/src/executor/physical_plans/physical_window_partition.rs +++ b/src/query/sql/src/executor/physical_plans/physical_window_partition.rs @@ -19,6 +19,7 @@ use databend_common_expression::DataSchemaRef; use crate::executor::explain::PlanStatsInfo; use crate::executor::physical_plans::SortDesc; use crate::executor::{IPhysicalPlan, PhysicalPlan, PhysicalPlanMeta}; +use crate::executor::physical_plan::PhysicalPlanDeriveHandle; use crate::IndexType; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] @@ -55,6 +56,20 @@ impl IPhysicalPlan for WindowPartition { fn try_find_single_data_source(&self) -> Option<&DataSourcePlan> { self.input.try_find_single_data_source() } + + fn derive_with(&self, handle: &mut Box) -> Box { + let derive_input = self.input.derive_with(handle); + + match handle.derive(self, vec![derive_input]) { + Ok(v) => v, + Err(children) => { + let mut new_window_partition = self.clone(); + assert_eq!(children.len(), 1); + new_window_partition.input = children[0]; + Box::new(new_window_partition) + } + } + } } #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] From b269460801eebc1caa85719ec435e274e8fddd71 Mon Sep 17 00:00:00 2001 From: zhang2014 Date: Mon, 30 Jun 2025 23:50:42 +0800 Subject: [PATCH 5/8] refactor(query): use trait to refactor physical plan --- .../interpreters/interpreter_index_refresh.rs | 24 ++- .../src/schedulers/fragments/fragmenter.rs | 1 - .../src/schedulers/fragments/plan_fragment.rs | 191 +++++++++--------- src/query/sql/src/executor/mod.rs | 4 +- src/query/sql/src/executor/physical_plan.rs | 13 +- .../sql/src/executor/physical_plan_builder.rs | 13 +- .../sql/src/executor/physical_plan_visitor.rs | 8 +- .../physical_add_stream_column.rs | 24 ++- .../physical_aggregate_expand.rs | 15 +- .../physical_aggregate_final.rs | 154 +++++++------- .../physical_aggregate_partial.rs | 20 +- .../physical_plans/physical_asof_join.rs | 3 +- .../physical_plans/physical_async_func.rs | 19 +- .../physical_plans/physical_broadcast.rs | 22 +- .../physical_plans/physical_cache_scan.rs | 11 +- .../physical_column_mutation.rs | 24 ++- .../physical_plans/physical_commit_sink.rs | 22 +- .../physical_plans/physical_compact_source.rs | 17 +- .../physical_constant_table_scan.rs | 11 +- .../physical_copy_into_location.rs | 18 +- .../physical_copy_into_table.rs | 55 +++-- .../physical_distributed_insert_select.rs | 19 +- .../physical_plans/physical_eval_scalar.rs | 42 ++-- .../physical_plans/physical_exchange.rs | 15 +- .../physical_plans/physical_exchange_sink.rs | 17 +- .../physical_exchange_source.rs | 9 +- .../physical_expression_scan.rs | 17 +- .../physical_plans/physical_filter.rs | 34 ++-- .../physical_plans/physical_hash_join.rs | 49 +++-- .../executor/physical_plans/physical_join.rs | 3 +- .../executor/physical_plans/physical_limit.rs | 18 +- .../physical_multi_table_insert.rs | 105 +++++++--- .../physical_plans/physical_mutation.rs | 25 ++- .../physical_mutation_into_organize.rs | 17 +- .../physical_mutation_into_split.rs | 18 +- .../physical_mutation_manipulate.rs | 17 +- .../physical_mutation_source.rs | 14 +- .../physical_plans/physical_project_set.rs | 19 +- .../physical_plans/physical_r_cte_scan.rs | 9 +- .../physical_plans/physical_range_join.rs | 25 ++- .../physical_plans/physical_recluster.rs | 25 ++- .../physical_replace_async_source.rs | 9 +- .../physical_replace_deduplicate.rs | 20 +- .../physical_plans/physical_replace_into.rs | 22 +- .../physical_plans/physical_row_fetch.rs | 17 +- .../executor/physical_plans/physical_sort.rs | 22 +- .../physical_plans/physical_table_scan.rs | 76 ++++--- .../executor/physical_plans/physical_udf.rs | 20 +- .../physical_plans/physical_union_all.rs | 19 +- .../physical_plans/physical_window.rs | 23 ++- .../physical_window_partition.rs | 17 +- 51 files changed, 903 insertions(+), 508 deletions(-) diff --git a/src/query/service/src/interpreters/interpreter_index_refresh.rs b/src/query/service/src/interpreters/interpreter_index_refresh.rs index ee4147d52ee1d..6f4ba43fce1a1 100644 --- a/src/query/service/src/interpreters/interpreter_index_refresh.rs +++ b/src/query/service/src/interpreters/interpreter_index_refresh.rs @@ -35,9 +35,10 @@ use databend_common_pipeline_transforms::processors::TransformPipelineHelper; use databend_common_sql::evaluator::BlockOperator; use databend_common_sql::evaluator::CompoundBlockOperator; use databend_common_sql::executor::physical_plans::TableScan; +use databend_common_sql::executor::IPhysicalPlan; use databend_common_sql::executor::PhysicalPlan; use databend_common_sql::executor::PhysicalPlanBuilder; -use databend_common_sql::executor::PhysicalPlanReplacer; +use databend_common_sql::executor::PhysicalPlanDeriveHandle; use databend_common_sql::plans::Plan; use databend_common_sql::plans::RefreshIndexPlan; use databend_common_sql::plans::RelOperator; @@ -377,14 +378,23 @@ async fn modify_last_update(ctx: Arc, req: UpdateIndexReq) -> Resu Ok(()) } -struct ReadSourceReplacer { +struct ReadSourceDeriveHandle { source: DataSourcePlan, } -impl PhysicalPlanReplacer for ReadSourceReplacer { - fn replace_table_scan(&mut self, plan: &TableScan) -> Result { - let mut plan = plan.clone(); - plan.source = Box::new(self.source.clone()); - Ok(PhysicalPlan::TableScan(plan)) +impl PhysicalPlanDeriveHandle for ReadSourceDeriveHandle { + fn derive( + &mut self, + v: &Box, + children: Vec>, + ) -> std::result::Result, Vec>> { + let Some(table_scan) = v.down_cast::() else { + return Err(children); + }; + + Ok(Box::new(TableScan { + source: Box::new(self.source.clone()), + ..table_scan.clone() + })) } } diff --git a/src/query/service/src/schedulers/fragments/fragmenter.rs b/src/query/service/src/schedulers/fragments/fragmenter.rs index 090f612c558d2..7f7cd877e87e8 100644 --- a/src/query/service/src/schedulers/fragments/fragmenter.rs +++ b/src/query/service/src/schedulers/fragments/fragmenter.rs @@ -31,7 +31,6 @@ use databend_common_sql::executor::physical_plans::Recluster; use databend_common_sql::executor::physical_plans::ReplaceInto; use databend_common_sql::executor::physical_plans::TableScan; use databend_common_sql::executor::physical_plans::UnionAll; -use databend_common_sql::executor::PhysicalPlanReplacer; use crate::clusters::ClusterHelper; use crate::schedulers::fragments::plan_fragment::FragmentType; diff --git a/src/query/service/src/schedulers/fragments/plan_fragment.rs b/src/query/service/src/schedulers/fragments/plan_fragment.rs index a3043937a0ed5..f11d1812f3b37 100644 --- a/src/query/service/src/schedulers/fragments/plan_fragment.rs +++ b/src/query/service/src/schedulers/fragments/plan_fragment.rs @@ -33,7 +33,8 @@ use databend_common_sql::executor::physical_plans::Recluster; use databend_common_sql::executor::physical_plans::ReplaceDeduplicate; use databend_common_sql::executor::physical_plans::ReplaceInto; use databend_common_sql::executor::physical_plans::TableScan; -use databend_common_sql::executor::{IPhysicalPlan, PhysicalPlanDeriveHandle}; +use databend_common_sql::executor::IPhysicalPlan; +use databend_common_sql::executor::PhysicalPlanDeriveHandle; use databend_common_storages_fuse::TableContext; use databend_storages_common_table_meta::meta::BlockSlotDescription; use databend_storages_common_table_meta::meta::Location; @@ -45,7 +46,6 @@ use crate::schedulers::QueryFragmentsActions; use crate::servers::flight::v1::exchange::DataExchange; use crate::sessions::QueryContext; use crate::sql::executor::PhysicalPlan; -use crate::sql::executor::PhysicalPlanReplacer; /// Type of plan fragment #[derive(Clone, Debug, PartialEq, Eq)] @@ -212,13 +212,11 @@ impl PlanFragment { } for (executor, sources) in executor_partitions { - let mut plan = self.plan.clone(); // Replace `ReadDataSourcePlan` with rewritten one and generate new fragment for it. - let mut replace_read_source = ReplaceReadSource { sources }; - plan = replace_read_source.replace(&plan)?; + let mut handle = ReadSourceDeriveHandle::new(sources); + let plan = self.plan.derive_with(&mut handle); - fragment_actions - .add_action(QueryFragmentAction::create(executor.clone(), plan.clone())); + fragment_actions.add_action(QueryFragmentAction::create(executor.clone(), plan)); } Ok(()) @@ -243,11 +241,8 @@ impl PlanFragment { let partition_reshuffle = partitions.reshuffle(executors)?; for (executor, parts) in partition_reshuffle.into_iter() { - let mut plan = self.plan.clone(); - - let mut replace_mutation_source = ReplaceMutationSource { partitions: parts }; - plan = replace_mutation_source.replace(&plan)?; - + let mut handle = MutationSourceDeriveHandle::new(parts); + let plan = self.plan.derive_with(&mut handle); fragment_actions.add_action(QueryFragmentAction::create(executor, plan)); } @@ -287,7 +282,6 @@ impl PlanFragment { // assign all the segment locations to each one of the executors, // but for each segment, one executor only need to take part of the blocks for (executor_idx, executor) in executors.into_iter().enumerate() { - let mut plan = self.plan.clone(); let need_insert = executor == local_id; let mut handle = ReplaceDeriveHandle::new( partitions.clone(), @@ -298,7 +292,7 @@ impl PlanFragment { need_insert, ); - plan = plan.derive_with(&mut handle); + let plan = self.plan.derive_with(&mut handle); fragment_actions.add_action(QueryFragmentAction::create(executor, plan)); } @@ -327,10 +321,8 @@ impl PlanFragment { let partition_reshuffle = partitions.reshuffle(executors)?; for (executor, parts) in partition_reshuffle.into_iter() { - let mut plan = self.plan.clone(); - let mut handle = CompactSourceDeriveHandle::new(parts); - plan = plan.derive_with(&mut handle); + let plan = self.plan.derive_with(&mut handle); fragment_actions.add_action(QueryFragmentAction::create(executor, plan)); } @@ -363,9 +355,8 @@ impl PlanFragment { let task_reshuffle = Self::reshuffle(executors, tasks)?; for (executor, tasks) in task_reshuffle.into_iter() { - let mut plan = self.plan.clone(); - let mut replace_recluster = ReplaceRecluster { tasks }; - plan = replace_recluster.replace(&plan)?; + let mut handle = ReclusterDeriveHandle::new(tasks); + let plan = self.plan.derive_with(&mut handle); fragment_actions.add_action(QueryFragmentAction::create(executor, plan)); } @@ -483,92 +474,105 @@ impl TryFrom for ConstTableColumn { } } -struct ReplaceReadSource { +struct ReadSourceDeriveHandle { sources: HashMap, } -impl PhysicalPlanReplacer for ReplaceReadSource { - fn replace_table_scan(&mut self, plan: &TableScan) -> Result { - let source = self.sources.remove(&plan.plan_id).ok_or_else(|| { - ErrorCode::Internal(format!( - "Cannot find data source for table scan plan {}", - plan.plan_id - )) - })?; - - let source = DataSourcePlan::try_from(source)?; - - Ok(PhysicalPlan::TableScan(TableScan { - plan_id: plan.plan_id, - scan_id: plan.scan_id, - source: Box::new(source), - name_mapping: plan.name_mapping.clone(), - table_index: plan.table_index, - stat_info: plan.stat_info.clone(), - internal_column: plan.internal_column.clone(), - })) - } - - fn replace_constant_table_scan(&mut self, plan: &ConstantTableScan) -> Result { - let source = self.sources.remove(&plan.plan_id).ok_or_else(|| { - ErrorCode::Internal(format!( - "Cannot find data source for constant table scan plan {}", - plan.plan_id - )) - })?; - - let const_table_columns = ConstTableColumn::try_from(source)?; - - Ok(PhysicalPlan::ConstantTableScan(ConstantTableScan { - plan_id: plan.plan_id, - values: const_table_columns.columns, - num_rows: const_table_columns.num_rows, - output_schema: plan.output_schema.clone(), - })) +impl ReadSourceDeriveHandle { + pub fn new(sources: HashMap) -> Box { + Box::new(ReadSourceDeriveHandle { sources }) } +} - fn replace_copy_into_table(&mut self, plan: &CopyIntoTable) -> Result { - match &plan.source { - CopyIntoTableSource::Query(query_physical_plan) => { - let input = self.replace(query_physical_plan)?; - Ok(PhysicalPlan::CopyIntoTable(Box::new(CopyIntoTable { - source: CopyIntoTableSource::Query(Box::new(input)), - ..plan.clone() - }))) - } - CopyIntoTableSource::Stage(v) => { - let input = self.replace(v)?; - Ok(PhysicalPlan::CopyIntoTable(Box::new(CopyIntoTable { - source: CopyIntoTableSource::Stage(Box::new(input)), - ..plan.clone() - }))) - } +impl PhysicalPlanDeriveHandle for ReadSourceDeriveHandle { + fn derive( + &mut self, + v: &Box, + children: Vec>, + ) -> std::result::Result, Vec>> { + if let Some(table_scan) = v.down_cast::() { + let source = self.sources.remove(&table_scan.get_id()).ok_or_else(|| { + ErrorCode::Internal(format!( + "Cannot find data source for table scan plan {}", + table_scan.get_id() + )) + })?; + + return Ok(Box::new(TableScan { + source: Box::new(DataSourcePlan::try_from(source)?), + ..table_scan.clone() + })); + } else if let Some(table_scan) = v.down_cast::() { + let source = self.sources.remove(&table_scan.get_id()).ok_or_else(|| { + ErrorCode::Internal(format!( + "Cannot find data source for constant table scan plan {}", + table_scan.get_id() + )) + })?; + + let const_table_columns = ConstTableColumn::try_from(source)?; + + return Ok(Box::new(ConstantTableScan { + values: const_table_columns.columns, + num_rows: const_table_columns.num_rows, + ..table_scan.clone() + })); } + + Err(children) } } -struct ReplaceRecluster { - pub tasks: Vec, +struct ReclusterDeriveHandle { + tasks: Vec, +} + +impl ReclusterDeriveHandle { + pub fn new(tasks: Vec) -> Box { + Box::new(ReclusterDeriveHandle { tasks }) + } } -impl PhysicalPlanReplacer for ReplaceRecluster { - fn replace_recluster(&mut self, plan: &Recluster) -> Result { - Ok(PhysicalPlan::Recluster(Box::new(Recluster { +impl PhysicalPlanDeriveHandle for ReclusterDeriveHandle { + fn derive( + &mut self, + v: &Box, + children: Vec>, + ) -> std::result::Result, Vec>> { + let Some(recluster) = v.down_cast::() else { + return Err(children); + }; + + Ok(Box::new(Recluster { tasks: self.tasks.clone(), - ..plan.clone() - }))) + ..recluster.clone() + })) } } -struct ReplaceMutationSource { - pub partitions: Partitions, +struct MutationSourceDeriveHandle { + partitions: Partitions, } -impl PhysicalPlanReplacer for ReplaceMutationSource { - fn replace_mutation_source(&mut self, plan: &MutationSource) -> Result { - Ok(PhysicalPlan::MutationSource(MutationSource { +impl MutationSourceDeriveHandle { + pub fn new(partitions: Partitions) -> Box { + Box::new(MutationSourceDeriveHandle { partitions }) + } +} + +impl PhysicalPlanDeriveHandle for MutationSourceDeriveHandle { + fn derive( + &mut self, + v: &Box, + children: Vec>, + ) -> std::result::Result, Vec>> { + let Some(mutation_source) = v.down_cast::() else { + return Err(children); + }; + + Ok(Box::new(MutationSource { partitions: self.partitions.clone(), - ..plan.clone() + ..mutation_source.clone() })) } } @@ -579,9 +583,7 @@ struct CompactSourceDeriveHandle { impl CompactSourceDeriveHandle { pub fn new(partitions: Partitions) -> Box { - Box::new(CompactSourceDeriveHandle { - partitions - }) + Box::new(CompactSourceDeriveHandle { partitions }) } } @@ -595,7 +597,6 @@ impl PhysicalPlanDeriveHandle for CompactSourceDeriveHandle { return Err(children); }; - Ok(Box::new(CompactSource { parts: self.partitions.clone(), ..compact_source.clone() @@ -611,7 +612,11 @@ struct ReplaceDeriveHandle { } impl ReplaceDeriveHandle { - pub fn new(partitions: Vec<(usize, Location)>, slot: Option, need_insert: bool) -> Box { + pub fn new( + partitions: Vec<(usize, Location)>, + slot: Option, + need_insert: bool, + ) -> Box { Box::new(ReplaceDeriveHandle { partitions, slot, diff --git a/src/query/sql/src/executor/mod.rs b/src/query/sql/src/executor/mod.rs index 13ae7a9c6f1c4..5b74fc81d1794 100644 --- a/src/query/sql/src/executor/mod.rs +++ b/src/query/sql/src/executor/mod.rs @@ -23,8 +23,9 @@ mod util; pub mod table_read_plan; pub use format::format_partial_tree; -pub use physical_plan::PhysicalPlan; pub use physical_plan::IPhysicalPlan; +pub use physical_plan::PhysicalPlan; +pub use physical_plan::PhysicalPlanDeriveHandle; pub use physical_plan::PhysicalPlanMeta; pub use physical_plan_builder::MutationBuildInfo; pub use physical_plan_builder::PhysicalPlanBuilder; @@ -33,4 +34,3 @@ pub use physical_plans::build_broadcast_plans; pub use physical_plans::PhysicalRuntimeFilter; pub use physical_plans::PhysicalRuntimeFilters; pub use util::*; -pub use physical_plan::PhysicalPlanDeriveHandle; diff --git a/src/query/sql/src/executor/physical_plan.rs b/src/query/sql/src/executor/physical_plan.rs index 28c5d33bc0fde..9a5ea7edcdfb6 100644 --- a/src/query/sql/src/executor/physical_plan.rs +++ b/src/query/sql/src/executor/physical_plan.rs @@ -15,6 +15,7 @@ use std::any::Any; use std::collections::HashMap; use std::fmt::Debug; + use databend_common_catalog::plan::DataSourceInfo; use databend_common_catalog::plan::DataSourcePlan; use databend_common_catalog::plan::PartStatistics; @@ -132,15 +133,15 @@ pub trait IPhysicalPlan: Debug { fn output_schema(&self) -> Result { match self.children().next() { None => Ok(DataSchemaRef::default()), - Some(child) => child.output_schema() + Some(child) => child.output_schema(), } } - fn children(&self) -> Box> + '_> { + fn children(&self) -> Box> + '_> { Box::new(std::iter::empty()) } - fn children_mut(&mut self) -> Box> + '_> { + fn children_mut(&mut self) -> Box> + '_> { Box::new(std::iter::empty()) } @@ -193,7 +194,8 @@ pub trait IPhysicalPlan: Debug { Ok(HashMap::new()) } - fn derive_with(&self, handle: &mut Box) -> Box; + fn derive_with(&self, handle: &mut Box) + -> Box; } pub trait PhysicalPlanExt { @@ -203,8 +205,7 @@ pub trait PhysicalPlanExt { } impl PhysicalPlanExt for T -where - T: 'static + IPhysicalPlan + Clone, +where T: 'static + IPhysicalPlan + Clone { fn clone_box(&self) -> Box { Box::new(self.clone()) diff --git a/src/query/sql/src/executor/physical_plan_builder.rs b/src/query/sql/src/executor/physical_plan_builder.rs index ee4850d20dda5..f03f51e71b8f5 100644 --- a/src/query/sql/src/executor/physical_plan_builder.rs +++ b/src/query/sql/src/executor/physical_plan_builder.rs @@ -25,7 +25,8 @@ use databend_storages_common_table_meta::meta::TableMetaTimestamps; use databend_storages_common_table_meta::meta::TableSnapshot; use crate::executor::explain::PlanStatsInfo; -use crate::executor::{IPhysicalPlan, PhysicalPlan}; +use crate::executor::IPhysicalPlan; +use crate::executor::PhysicalPlan; use crate::optimizer::ir::RelExpr; use crate::optimizer::ir::SExpr; use crate::plans::RelOperator; @@ -62,7 +63,11 @@ impl PhysicalPlanBuilder { }) } - pub async fn build(&mut self, s_expr: &SExpr, required: ColumnSet) -> Result> { + pub async fn build( + &mut self, + s_expr: &SExpr, + required: ColumnSet, + ) -> Result> { let mut plan = self.build_physical_plan(s_expr, required).await?; plan.adjust_plan_id(&mut 0); @@ -96,7 +101,9 @@ impl PhysicalPlanBuilder { } RelOperator::Sort(sort) => self.build_sort(s_expr, sort, required, stat_info).await, RelOperator::Limit(limit) => self.build_limit(s_expr, limit, required, stat_info).await, - RelOperator::Exchange(exchange) => self.build_exchange(s_expr, exchange, required).await, + RelOperator::Exchange(exchange) => { + self.build_exchange(s_expr, exchange, required).await + } RelOperator::UnionAll(union_all) => { self.build_union_all(s_expr, union_all, required, stat_info) .await diff --git a/src/query/sql/src/executor/physical_plan_visitor.rs b/src/query/sql/src/executor/physical_plan_visitor.rs index 41d3bbcf1512d..76d4e2944fd7f 100644 --- a/src/query/sql/src/executor/physical_plan_visitor.rs +++ b/src/query/sql/src/executor/physical_plan_visitor.rs @@ -13,7 +13,7 @@ // limitations under the License. use databend_common_exception::Result; -use crate::executor::{IPhysicalPlan, PhysicalPlanMeta}; + use super::physical_plans::AddStreamColumn; use super::physical_plans::BroadcastSink; use super::physical_plans::BroadcastSource; @@ -24,7 +24,8 @@ use super::physical_plans::MutationManipulate; use super::physical_plans::MutationOrganize; use super::physical_plans::MutationSplit; use super::physical_plans::RecursiveCteScan; -use crate::executor::physical_plan::{PhysicalPlan, PhysicalPlanExt}; +use crate::executor::physical_plan::PhysicalPlan; +use crate::executor::physical_plan::PhysicalPlanExt; use crate::executor::physical_plans::AggregateExpand; use crate::executor::physical_plans::AggregateFinal; use crate::executor::physical_plans::AggregatePartial; @@ -68,7 +69,8 @@ use crate::executor::physical_plans::Udf; use crate::executor::physical_plans::UnionAll; use crate::executor::physical_plans::Window; use crate::executor::physical_plans::WindowPartition; -// +use crate::executor::IPhysicalPlan; +use crate::executor::PhysicalPlanMeta; // pub trait PhysicalPlanReplacer { // fn replace(&mut self, plan: &PhysicalPlan) -> Result { // match plan { diff --git a/src/query/sql/src/executor/physical_plans/physical_add_stream_column.rs b/src/query/sql/src/executor/physical_plans/physical_add_stream_column.rs index 3aadcdb7295ba..2476871e99ccd 100644 --- a/src/query/sql/src/executor/physical_plans/physical_add_stream_column.rs +++ b/src/query/sql/src/executor/physical_plans/physical_add_stream_column.rs @@ -23,7 +23,11 @@ use databend_common_expression::ORIGIN_BLOCK_ID_COL_NAME; use databend_common_expression::ORIGIN_BLOCK_ROW_NUM_COL_NAME; use databend_common_expression::ORIGIN_VERSION_COL_NAME; -use crate::executor::{IPhysicalPlan, PhysicalPlan, PhysicalPlanMeta}; +use crate::executor::physical_plan::DeriveExt; +use crate::executor::physical_plan::PhysicalPlanDeriveHandle; +use crate::executor::IPhysicalPlan; +use crate::executor::PhysicalPlan; +use crate::executor::PhysicalPlanMeta; use crate::planner::CURRENT_BLOCK_ID_COL_NAME; use crate::planner::CURRENT_BLOCK_ROW_NUM_COL_NAME; use crate::plans::BoundColumnRef; @@ -31,7 +35,6 @@ use crate::plans::ConstantExpr; use crate::plans::FunctionCall; use crate::Binder; use crate::ColumnBindingBuilder; -use crate::executor::physical_plan::{DeriveExt, PhysicalPlanDeriveHandle}; use crate::MetadataRef; use crate::ScalarExpr; use crate::Visibility; @@ -55,15 +58,20 @@ impl IPhysicalPlan for AddStreamColumn { &mut self.meta } - fn children<'a>(&'a self) -> Box> + 'a> { + fn children<'a>(&'a self) -> Box> + 'a> { Box::new(std::iter::once(&self.input)) } - fn children_mut<'a>(&'a mut self) -> Box> + 'a> { + fn children_mut<'a>( + &'a mut self, + ) -> Box> + 'a> { Box::new(std::iter::once(&mut self.input)) } - fn derive_with(&self, handle: &mut Box) -> Box { + fn derive_with( + &self, + handle: &mut Box, + ) -> Box { let derive_input = self.input.derive_with(handle); match handle.derive(self, vec![derive_input]) { @@ -113,7 +121,7 @@ impl AddStreamColumn { Box::new(stream_column.data_type()), Visibility::Visible, ) - .build(), + .build(), }); let current_stream_column_scalar_expr = match stream_column.column_type() { @@ -134,7 +142,7 @@ impl AddStreamColumn { Box::new(stream_column.data_type()), Visibility::Visible, ) - .build(), + .build(), }) } StreamColumnType::OriginRowNum => { @@ -147,7 +155,7 @@ impl AddStreamColumn { Box::new(stream_column.data_type()), Visibility::Visible, ) - .build(), + .build(), }) } StreamColumnType::RowVersion => unreachable!(), diff --git a/src/query/sql/src/executor/physical_plans/physical_aggregate_expand.rs b/src/query/sql/src/executor/physical_plans/physical_aggregate_expand.rs index eda2e047e4853..80dd78b526c3f 100644 --- a/src/query/sql/src/executor/physical_plans/physical_aggregate_expand.rs +++ b/src/query/sql/src/executor/physical_plans/physical_aggregate_expand.rs @@ -21,8 +21,10 @@ use databend_common_expression::DataSchemaRef; use databend_common_expression::DataSchemaRefExt; use crate::executor::explain::PlanStatsInfo; -use crate::executor::{IPhysicalPlan, PhysicalPlan, PhysicalPlanMeta}; use crate::executor::physical_plan::PhysicalPlanDeriveHandle; +use crate::executor::IPhysicalPlan; +use crate::executor::PhysicalPlan; +use crate::executor::PhysicalPlanMeta; use crate::plans::GroupingSets; use crate::IndexType; @@ -75,11 +77,13 @@ impl IPhysicalPlan for AggregateExpand { Ok(DataSchemaRefExt::create(output_fields)) } - fn children<'a>(&'a self) -> Box> + 'a> { + fn children<'a>(&'a self) -> Box> + 'a> { Box::new(std::iter::once(&self.input)) } - fn children_mut<'a>(&'a mut self) -> Box> + 'a> { + fn children_mut<'a>( + &'a mut self, + ) -> Box> + 'a> { Box::new(std::iter::once(&mut self.input)) } @@ -99,7 +103,10 @@ impl IPhysicalPlan for AggregateExpand { .join(", ")) } - fn derive_with(&self, handle: &mut Box) -> Box { + fn derive_with( + &self, + handle: &mut Box, + ) -> Box { let derive_input = self.input.derive_with(handle); match handle.derive(self, vec![derive_input]) { diff --git a/src/query/sql/src/executor/physical_plans/physical_aggregate_final.rs b/src/query/sql/src/executor/physical_plans/physical_aggregate_final.rs index 0c94ce2a3c9e9..8fa3ae507482a 100644 --- a/src/query/sql/src/executor/physical_plans/physical_aggregate_final.rs +++ b/src/query/sql/src/executor/physical_plans/physical_aggregate_final.rs @@ -14,7 +14,7 @@ use std::collections::HashMap; use std::sync::Arc; -use itertools::Itertools; + use databend_common_catalog::plan::DataSourcePlan; use databend_common_exception::ErrorCode; use databend_common_exception::Result; @@ -22,21 +22,24 @@ use databend_common_expression::DataField; use databend_common_expression::DataSchemaRef; use databend_common_expression::DataSchemaRefExt; use databend_common_expression::RemoteExpr; +use itertools::Itertools; use super::SortDesc; use crate::executor::explain::PlanStatsInfo; +use crate::executor::physical_plan::PhysicalPlanDeriveHandle; use crate::executor::physical_plans::AggregateExpand; use crate::executor::physical_plans::AggregateFunctionDesc; use crate::executor::physical_plans::AggregateFunctionSignature; use crate::executor::physical_plans::AggregatePartial; use crate::executor::physical_plans::Exchange; -use crate::executor::{IPhysicalPlan, PhysicalPlan, PhysicalPlanMeta}; +use crate::executor::IPhysicalPlan; +use crate::executor::PhysicalPlan; use crate::executor::PhysicalPlanBuilder; +use crate::executor::PhysicalPlanMeta; use crate::optimizer::ir::SExpr; use crate::plans::AggregateMode; use crate::plans::DummyTableScan; use crate::ColumnSet; -use crate::executor::physical_plan::PhysicalPlanDeriveHandle; use crate::IndexType; use crate::ScalarExpr; @@ -80,11 +83,13 @@ impl IPhysicalPlan for AggregateFinal { Ok(DataSchemaRefExt::create(fields)) } - fn children<'a>(&'a self) -> Box> + 'a> { + fn children<'a>(&'a self) -> Box> + 'a> { Box::new(std::iter::once(&self.input)) } - fn children_mut<'a>(&'a mut self) -> Box> + 'a> { + fn children_mut<'a>( + &'a mut self, + ) -> Box> + 'a> { Box::new(std::iter::once(&mut self.input)) } @@ -108,7 +113,10 @@ impl IPhysicalPlan for AggregateFinal { Ok(labels) } - fn derive_with(&self, handle: &mut Box) -> Box { + fn derive_with( + &self, + handle: &mut Box, + ) -> Box { let derive_input = self.input.derive_with(handle); match handle.derive(self, vec![derive_input]) { @@ -286,12 +294,12 @@ impl PhysicalPlanBuilder { if let Some(grouping_sets) = agg.grouping_sets.as_ref() { assert_eq!(grouping_sets.dup_group_items.len(), group_items.len() - 1); // ignore `_grouping_id`. - // If the aggregation function argument if a group item, - // we cannot use the group item directly. - // It's because the group item will be wrapped with nullable and fill dummy NULLs (in `AggregateExpand` plan), - // which will cause panic while executing aggregation function. - // To avoid the panic, we will duplicate (`Arc::clone`) original group item columns in `AggregateExpand`, - // we should use these columns instead. + // If the aggregation function argument if a group item, + // we cannot use the group item directly. + // It's because the group item will be wrapped with nullable and fill dummy NULLs (in `AggregateExpand` plan), + // which will cause panic while executing aggregation function. + // To avoid the panic, we will duplicate (`Arc::clone`) original group item columns in `AggregateExpand`, + // we should use these columns instead. for func in agg_funcs.iter_mut() { for arg in func.arg_indices.iter_mut() { if let Some(pos) = group_items.iter().position(|g| g == arg) { @@ -316,63 +324,63 @@ impl PhysicalPlanBuilder { match input { PhysicalPlan::Exchange(Exchange { input, kind, .. }) - if group_by_shuffle_mode == "before_merge" => - { - let aggregate_partial = if let Some(grouping_sets) = agg.grouping_sets { - let expand = AggregateExpand { - input, - grouping_sets, - group_bys: group_items.clone(), - stat_info: Some(stat_info.clone()), - meta: PhysicalPlanMeta::new("AggregateExpand"), - }; - - AggregatePartial { - input: Box::new(PhysicalPlan::AggregateExpand(expand)), - agg_funcs, - enable_experimental_aggregate_hashtable, - group_by_display, - group_by: group_items, - stat_info: Some(stat_info), - rank_limit: None, - meta: PhysicalPlanMeta::new("AggregatePartial"), - } - } else { - AggregatePartial { - input, - agg_funcs, - rank_limit, - group_by_display, - enable_experimental_aggregate_hashtable, - group_by: group_items, - stat_info: Some(stat_info), - meta: PhysicalPlanMeta::new("AggregatePartial"), - } - }; - - let keys = { - let schema = aggregate_partial.output_schema()?; - let end = schema.num_fields(); - let start = end - aggregate_partial.group_by.len(); - (start..end) - .map(|id| RemoteExpr::ColumnRef { - span: None, - id, - data_type: schema.field(id).data_type().clone(), - display_name: (id - start).to_string(), - }) - .collect() + if group_by_shuffle_mode == "before_merge" => + { + let aggregate_partial = if let Some(grouping_sets) = agg.grouping_sets { + let expand = AggregateExpand { + input, + grouping_sets, + group_bys: group_items.clone(), + stat_info: Some(stat_info.clone()), + meta: PhysicalPlanMeta::new("AggregateExpand"), }; - PhysicalPlan::Exchange(Exchange { - kind, - keys, - ignore_exchange: false, - allow_adjust_parallelism: true, - meta: PhysicalPlanMeta::new("Exchange"), - input: Box::new(PhysicalPlan::AggregatePartial(aggregate_partial)), - }) - } + AggregatePartial { + input: Box::new(PhysicalPlan::AggregateExpand(expand)), + agg_funcs, + enable_experimental_aggregate_hashtable, + group_by_display, + group_by: group_items, + stat_info: Some(stat_info), + rank_limit: None, + meta: PhysicalPlanMeta::new("AggregatePartial"), + } + } else { + AggregatePartial { + input, + agg_funcs, + rank_limit, + group_by_display, + enable_experimental_aggregate_hashtable, + group_by: group_items, + stat_info: Some(stat_info), + meta: PhysicalPlanMeta::new("AggregatePartial"), + } + }; + + let keys = { + let schema = aggregate_partial.output_schema()?; + let end = schema.num_fields(); + let start = end - aggregate_partial.group_by.len(); + (start..end) + .map(|id| RemoteExpr::ColumnRef { + span: None, + id, + data_type: schema.field(id).data_type().clone(), + display_name: (id - start).to_string(), + }) + .collect() + }; + + PhysicalPlan::Exchange(Exchange { + kind, + keys, + ignore_exchange: false, + allow_adjust_parallelism: true, + meta: PhysicalPlanMeta::new("Exchange"), + input: Box::new(PhysicalPlan::AggregatePartial(aggregate_partial)), + }) + } _ => { if let Some(grouping_sets) = agg.grouping_sets { let expand = AggregateExpand { @@ -415,9 +423,9 @@ impl PhysicalPlanBuilder { PhysicalPlan::AggregatePartial(ref agg) => agg.input.output_schema()?, PhysicalPlan::Exchange(Exchange { - input: box PhysicalPlan::AggregatePartial(ref agg), - .. - }) => agg.input.output_schema()?, + input: box PhysicalPlan::AggregatePartial(ref agg), + .. + }) => agg.input.output_schema()?, _ => { return Err(ErrorCode::Internal(format!( @@ -562,9 +570,9 @@ impl PhysicalPlanBuilder { } PhysicalPlan::Exchange(Exchange { - input: box PhysicalPlan::AggregatePartial(ref partial), - .. - }) => { + input: box PhysicalPlan::AggregatePartial(ref partial), + .. + }) => { let before_group_by_schema = partial.input.output_schema()?; Box::new(AggregateFinal { diff --git a/src/query/sql/src/executor/physical_plans/physical_aggregate_partial.rs b/src/query/sql/src/executor/physical_plans/physical_aggregate_partial.rs index 3b3e0763b81a7..24d0e103ae439 100644 --- a/src/query/sql/src/executor/physical_plans/physical_aggregate_partial.rs +++ b/src/query/sql/src/executor/physical_plans/physical_aggregate_partial.rs @@ -13,7 +13,7 @@ // limitations under the License. use std::collections::HashMap; -use itertools::Itertools; + use databend_common_catalog::plan::DataSourcePlan; use databend_common_exception::Result; use databend_common_expression::types::DataType; @@ -22,12 +22,15 @@ use databend_common_expression::DataBlock; use databend_common_expression::DataField; use databend_common_expression::DataSchemaRef; use databend_common_expression::DataSchemaRefExt; +use itertools::Itertools; use super::SortDesc; use crate::executor::explain::PlanStatsInfo; -use crate::executor::physical_plans::common::AggregateFunctionDesc; -use crate::executor::{IPhysicalPlan, PhysicalPlan, PhysicalPlanMeta}; use crate::executor::physical_plan::PhysicalPlanDeriveHandle; +use crate::executor::physical_plans::common::AggregateFunctionDesc; +use crate::executor::IPhysicalPlan; +use crate::executor::PhysicalPlan; +use crate::executor::PhysicalPlanMeta; use crate::IndexType; #[derive(serde::Serialize, serde::Deserialize, Clone, Debug)] @@ -76,11 +79,13 @@ impl IPhysicalPlan for AggregatePartial { Ok(DataSchemaRefExt::create(fields)) } - fn children<'a>(&'a self) -> Box> + 'a> { + fn children<'a>(&'a self) -> Box> + 'a> { Box::new(std::iter::once(&self.input)) } - fn children_mut<'a>(&'a mut self) -> Box> + 'a> { + fn children_mut<'a>( + &'a mut self, + ) -> Box> + 'a> { Box::new(std::iter::once(&mut self.input)) } @@ -105,7 +110,10 @@ impl IPhysicalPlan for AggregatePartial { Ok(labels) } - fn derive_with(&self, handle: &mut Box) -> Box { + fn derive_with( + &self, + handle: &mut Box, + ) -> Box { let derive_input = self.input.derive_with(handle); match handle.derive(self, vec![derive_input]) { diff --git a/src/query/sql/src/executor/physical_plans/physical_asof_join.rs b/src/query/sql/src/executor/physical_plans/physical_asof_join.rs index eacc5747981e3..7dfbdcea85b6c 100644 --- a/src/query/sql/src/executor/physical_plans/physical_asof_join.rs +++ b/src/query/sql/src/executor/physical_plans/physical_asof_join.rs @@ -24,7 +24,8 @@ use crate::binder::bind_window_function_info; use crate::binder::ColumnBindingBuilder; use crate::binder::WindowFunctionInfo; use crate::binder::WindowOrderByInfo; -use crate::executor::{IPhysicalPlan, PhysicalPlan}; +use crate::executor::IPhysicalPlan; +use crate::executor::PhysicalPlan; use crate::executor::PhysicalPlanBuilder; use crate::optimizer::ir::RelExpr; use crate::optimizer::ir::SExpr; diff --git a/src/query/sql/src/executor/physical_plans/physical_async_func.rs b/src/query/sql/src/executor/physical_plans/physical_async_func.rs index a2e2e8ce76b92..f481b69806357 100644 --- a/src/query/sql/src/executor/physical_plans/physical_async_func.rs +++ b/src/query/sql/src/executor/physical_plans/physical_async_func.rs @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -use itertools::Itertools; use databend_common_catalog::plan::DataSourcePlan; use databend_common_exception::ErrorCode; use databend_common_exception::Result; @@ -20,14 +19,17 @@ use databend_common_expression::types::DataType; use databend_common_expression::DataField; use databend_common_expression::DataSchemaRef; use databend_common_expression::DataSchemaRefExt; +use itertools::Itertools; use crate::executor::explain::PlanStatsInfo; -use crate::executor::{IPhysicalPlan, PhysicalPlan, PhysicalPlanMeta}; +use crate::executor::physical_plan::PhysicalPlanDeriveHandle; +use crate::executor::IPhysicalPlan; +use crate::executor::PhysicalPlan; use crate::executor::PhysicalPlanBuilder; +use crate::executor::PhysicalPlanMeta; use crate::optimizer::ir::SExpr; use crate::plans::AsyncFunctionArgument; use crate::ColumnSet; -use crate::executor::physical_plan::PhysicalPlanDeriveHandle; use crate::IndexType; use crate::ScalarExpr; @@ -61,11 +63,13 @@ impl IPhysicalPlan for AsyncFunction { Ok(DataSchemaRefExt::create(fields)) } - fn children<'a>(&'a self) -> Box> + 'a> { + fn children<'a>(&'a self) -> Box> + 'a> { Box::new(std::iter::once(&self.input)) } - fn children_mut<'a>(&'a mut self) -> Box> + 'a> { + fn children_mut<'a>( + &'a mut self, + ) -> Box> + 'a> { Box::new(std::iter::once(&mut self.input)) } @@ -82,7 +86,10 @@ impl IPhysicalPlan for AsyncFunction { .join(", ")) } - fn derive_with(&self, handle: &mut Box) -> Box { + fn derive_with( + &self, + handle: &mut Box, + ) -> Box { let derive_input = self.input.derive_with(handle); match handle.derive(self, vec![derive_input]) { diff --git a/src/query/sql/src/executor/physical_plans/physical_broadcast.rs b/src/query/sql/src/executor/physical_plans/physical_broadcast.rs index 2014b885aa3a7..e472836d1dd8c 100644 --- a/src/query/sql/src/executor/physical_plans/physical_broadcast.rs +++ b/src/query/sql/src/executor/physical_plans/physical_broadcast.rs @@ -15,10 +15,13 @@ use databend_common_catalog::table_context::TableContext; use databend_common_exception::Result; use databend_common_expression::DataSchemaRef; + use super::Exchange; use super::FragmentKind; -use crate::executor::{IPhysicalPlan, PhysicalPlan, PhysicalPlanMeta}; use crate::executor::physical_plan::PhysicalPlanDeriveHandle; +use crate::executor::IPhysicalPlan; +use crate::executor::PhysicalPlan; +use crate::executor::PhysicalPlanMeta; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct BroadcastSource { @@ -36,7 +39,10 @@ impl IPhysicalPlan for BroadcastSource { &mut self.meta } - fn derive_with(&self, handle: &mut Box) -> Box { + fn derive_with( + &self, + handle: &mut Box, + ) -> Box { match handle.derive(self, vec![]) { Ok(v) => v, Err(children) => { @@ -45,7 +51,6 @@ impl IPhysicalPlan for BroadcastSource { } } } - } #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] @@ -69,15 +74,20 @@ impl IPhysicalPlan for BroadcastSink { Ok(DataSchemaRef::default()) } - fn children<'a>(&'a self) -> Box> + 'a> { + fn children<'a>(&'a self) -> Box> + 'a> { Box::new(std::iter::once(&self.input)) } - fn children_mut<'a>(&'a mut self) -> Box> + 'a> { + fn children_mut<'a>( + &'a mut self, + ) -> Box> + 'a> { Box::new(std::iter::once(&mut self.input)) } - fn derive_with(&self, handle: &mut Box) -> Box { + fn derive_with( + &self, + handle: &mut Box, + ) -> Box { let derive_input = self.input.derive_with(handle); match handle.derive(self, vec![derive_input]) { diff --git a/src/query/sql/src/executor/physical_plans/physical_cache_scan.rs b/src/query/sql/src/executor/physical_plans/physical_cache_scan.rs index 4cfe110d8895e..5d21e033e48ae 100644 --- a/src/query/sql/src/executor/physical_plans/physical_cache_scan.rs +++ b/src/query/sql/src/executor/physical_plans/physical_cache_scan.rs @@ -16,11 +16,13 @@ use databend_common_exception::Result; use databend_common_expression::DataSchemaRef; use databend_common_expression::DataSchemaRefExt; -use crate::executor::{IPhysicalPlan, PhysicalPlan, PhysicalPlanMeta}; +use crate::executor::physical_plan::PhysicalPlanDeriveHandle; +use crate::executor::IPhysicalPlan; +use crate::executor::PhysicalPlan; use crate::executor::PhysicalPlanBuilder; +use crate::executor::PhysicalPlanMeta; use crate::plans::CacheSource; use crate::ColumnSet; -use crate::executor::physical_plan::PhysicalPlanDeriveHandle; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct CacheScan { @@ -43,7 +45,10 @@ impl IPhysicalPlan for CacheScan { Ok(self.output_schema.clone()) } - fn derive_with(&self, handle: &mut Box) -> Box { + fn derive_with( + &self, + handle: &mut Box, + ) -> Box { match handle.derive(self, vec![]) { Ok(v) => v, Err(children) => { diff --git a/src/query/sql/src/executor/physical_plans/physical_column_mutation.rs b/src/query/sql/src/executor/physical_plans/physical_column_mutation.rs index f01427a43c663..8c26c2a21ba3c 100644 --- a/src/query/sql/src/executor/physical_plans/physical_column_mutation.rs +++ b/src/query/sql/src/executor/physical_plans/physical_column_mutation.rs @@ -13,14 +13,19 @@ // limitations under the License. use std::collections::HashMap; + use databend_common_catalog::plan::DataSourcePlan; -use databend_common_expression::{DataSchemaRef, RemoteExpr}; +use databend_common_exception::Result; +use databend_common_expression::DataSchemaRef; +use databend_common_expression::RemoteExpr; use databend_common_meta_app::schema::TableInfo; use databend_storages_common_table_meta::meta::TableMetaTimestamps; -use crate::executor::{IPhysicalPlan, PhysicalPlanMeta}; -use crate::executor::physical_plan::{PhysicalPlanDeriveHandle, PhysicalPlan}; + +use crate::executor::physical_plan::PhysicalPlan; +use crate::executor::physical_plan::PhysicalPlanDeriveHandle; use crate::executor::physical_plans::MutationKind; -use databend_common_exception::Result; +use crate::executor::IPhysicalPlan; +use crate::executor::PhysicalPlanMeta; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct ColumnMutation { @@ -50,15 +55,20 @@ impl IPhysicalPlan for ColumnMutation { Ok(DataSchemaRef::default()) } - fn children<'a>(&'a self) -> Box> + 'a> { + fn children<'a>(&'a self) -> Box> + 'a> { Box::new(std::iter::once(&self.input)) } - fn children_mut<'a>(&'a mut self) -> Box> + 'a> { + fn children_mut<'a>( + &'a mut self, + ) -> Box> + 'a> { Box::new(std::iter::once(&mut self.input)) } - fn derive_with(&self, handle: &mut Box) -> Box { + fn derive_with( + &self, + handle: &mut Box, + ) -> Box { let derive_input = self.input.derive_with(handle); match handle.derive(self, vec![derive_input]) { diff --git a/src/query/sql/src/executor/physical_plans/physical_commit_sink.rs b/src/query/sql/src/executor/physical_plans/physical_commit_sink.rs index b936a7120e8ae..4c1fed220a2d3 100644 --- a/src/query/sql/src/executor/physical_plans/physical_commit_sink.rs +++ b/src/query/sql/src/executor/physical_plans/physical_commit_sink.rs @@ -14,18 +14,21 @@ use std::sync::Arc; -use databend_common_catalog::plan::{DataSourcePlan, ReclusterInfoSideCar}; +use databend_common_catalog::plan::DataSourcePlan; +use databend_common_catalog::plan::ReclusterInfoSideCar; +use databend_common_exception::Result; use databend_common_expression::DataSchemaRef; use databend_common_meta_app::schema::TableInfo; use databend_common_meta_app::schema::UpdateStreamMetaReq; use databend_storages_common_table_meta::meta::TableMetaTimestamps; use databend_storages_common_table_meta::meta::TableSnapshot; +use crate::executor::physical_plan::PhysicalPlanDeriveHandle; use crate::executor::physical_plans::common::MutationKind; -use crate::executor::{IPhysicalPlan, PhysicalPlan, PhysicalPlanMeta}; +use crate::executor::IPhysicalPlan; +use crate::executor::PhysicalPlan; +use crate::executor::PhysicalPlanMeta; use crate::plans::TruncateMode; -use databend_common_exception::Result; -use crate::executor::physical_plan::PhysicalPlanDeriveHandle; // serde is required by `PhysicalPlan` /// The commit sink is used to commit the data to the table. @@ -58,15 +61,20 @@ impl IPhysicalPlan for CommitSink { Ok(DataSchemaRef::default()) } - fn children<'a>(&'a self) -> Box> + 'a> { + fn children<'a>(&'a self) -> Box> + 'a> { Box::new(std::iter::once(&self.input)) } - fn children_mut<'a>(&'a mut self) -> Box> + 'a> { + fn children_mut<'a>( + &'a mut self, + ) -> Box> + 'a> { Box::new(std::iter::once(&mut self.input)) } - fn derive_with(&self, handle: &mut Box) -> Box { + fn derive_with( + &self, + handle: &mut Box, + ) -> Box { let derive_input = self.input.derive_with(handle); match handle.derive(self, vec![derive_input]) { diff --git a/src/query/sql/src/executor/physical_plans/physical_compact_source.rs b/src/query/sql/src/executor/physical_plans/physical_compact_source.rs index 31900011978e5..f9f4666c308ca 100644 --- a/src/query/sql/src/executor/physical_plans/physical_compact_source.rs +++ b/src/query/sql/src/executor/physical_plans/physical_compact_source.rs @@ -14,23 +14,27 @@ use std::collections::HashSet; -use databend_common_catalog::plan::{DataSourcePlan, PartInfoType}; +use databend_common_catalog::plan::DataSourcePlan; +use databend_common_catalog::plan::PartInfoType; use databend_common_catalog::plan::Partitions; use databend_common_catalog::table::TableExt; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::{ColumnId, DataSchemaRef}; +use databend_common_expression::ColumnId; +use databend_common_expression::DataSchemaRef; use databend_common_meta_app::schema::TableInfo; use databend_storages_common_table_meta::meta::TableMetaTimestamps; +use crate::executor::physical_plan::PhysicalPlanDeriveHandle; use crate::executor::physical_plans::CommitSink; use crate::executor::physical_plans::CommitType; use crate::executor::physical_plans::Exchange; use crate::executor::physical_plans::FragmentKind; use crate::executor::physical_plans::MutationKind; -use crate::executor::{IPhysicalPlan, PhysicalPlan, PhysicalPlanMeta}; -use crate::executor::physical_plan::PhysicalPlanDeriveHandle; +use crate::executor::IPhysicalPlan; +use crate::executor::PhysicalPlan; use crate::executor::PhysicalPlanBuilder; +use crate::executor::PhysicalPlanMeta; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct CompactSource { @@ -51,7 +55,10 @@ impl IPhysicalPlan for CompactSource { &mut self.meta } - fn derive_with(&self, handle: &mut Box) -> Box { + fn derive_with( + &self, + handle: &mut Box, + ) -> Box { match handle.derive(self, vec![]) { Ok(v) => v, Err(children) => { diff --git a/src/query/sql/src/executor/physical_plans/physical_constant_table_scan.rs b/src/query/sql/src/executor/physical_plans/physical_constant_table_scan.rs index c411512ed8c32..41623b0c580cb 100644 --- a/src/query/sql/src/executor/physical_plans/physical_constant_table_scan.rs +++ b/src/query/sql/src/executor/physical_plans/physical_constant_table_scan.rs @@ -16,10 +16,12 @@ use databend_common_exception::Result; use databend_common_expression::Column; use databend_common_expression::DataSchemaRef; -use crate::executor::{IPhysicalPlan, PhysicalPlan, PhysicalPlanMeta}; +use crate::executor::physical_plan::PhysicalPlanDeriveHandle; +use crate::executor::IPhysicalPlan; +use crate::executor::PhysicalPlan; use crate::executor::PhysicalPlanBuilder; +use crate::executor::PhysicalPlanMeta; use crate::ColumnSet; -use crate::executor::physical_plan::PhysicalPlanDeriveHandle; use crate::IndexType; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] @@ -44,7 +46,10 @@ impl IPhysicalPlan for ConstantTableScan { Ok(self.output_schema.clone()) } - fn derive_with(&self, handle: &mut Box) -> Box { + fn derive_with( + &self, + handle: &mut Box, + ) -> Box { match handle.derive(self, vec![]) { Ok(v) => v, Err(children) => { diff --git a/src/query/sql/src/executor/physical_plans/physical_copy_into_location.rs b/src/query/sql/src/executor/physical_plans/physical_copy_into_location.rs index 387bb848a4d54..c14d38fc6bce9 100644 --- a/src/query/sql/src/executor/physical_plans/physical_copy_into_location.rs +++ b/src/query/sql/src/executor/physical_plans/physical_copy_into_location.rs @@ -22,9 +22,11 @@ use databend_common_expression::DataSchemaRefExt; use databend_common_expression::TableSchemaRef; use databend_storages_common_stage::CopyIntoLocationInfo; -use crate::executor::{IPhysicalPlan, PhysicalPlan, PhysicalPlanMeta}; -use crate::ColumnBinding; use crate::executor::physical_plan::PhysicalPlanDeriveHandle; +use crate::executor::IPhysicalPlan; +use crate::executor::PhysicalPlan; +use crate::executor::PhysicalPlanMeta; +use crate::ColumnBinding; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct CopyIntoLocation { @@ -54,11 +56,13 @@ impl IPhysicalPlan for CopyIntoLocation { ])) } - fn children<'a>(&'a self) -> Box> + 'a> { + fn children<'a>(&'a self) -> Box> + 'a> { Box::new(std::iter::once(&self.input)) } - fn children_mut<'a>(&'a mut self) -> Box> + 'a> { + fn children_mut<'a>( + &'a mut self, + ) -> Box> + 'a> { Box::new(std::iter::once(&mut self.input)) } @@ -67,7 +71,10 @@ impl IPhysicalPlan for CopyIntoLocation { self.input.try_find_single_data_source() } - fn derive_with(&self, handle: &mut Box) -> Box { + fn derive_with( + &self, + handle: &mut Box, + ) -> Box { let derive_input = self.input.derive_with(handle); match handle.derive(self, vec![derive_input]) { @@ -80,5 +87,4 @@ impl IPhysicalPlan for CopyIntoLocation { } } } - } diff --git a/src/query/sql/src/executor/physical_plans/physical_copy_into_table.rs b/src/query/sql/src/executor/physical_plans/physical_copy_into_table.rs index 9d052690d3b6e..0494650a6722b 100644 --- a/src/query/sql/src/executor/physical_plans/physical_copy_into_table.rs +++ b/src/query/sql/src/executor/physical_plans/physical_copy_into_table.rs @@ -12,7 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -use databend_common_catalog::plan::{DataSourcePlan, StageTableInfo}; +use databend_common_catalog::plan::DataSourcePlan; +use databend_common_catalog::plan::StageTableInfo; use databend_common_exception::Result; use databend_common_expression::DataSchemaRef; use databend_common_expression::DataSchemaRefExt; @@ -21,11 +22,13 @@ use databend_common_meta_app::schema::TableInfo; use databend_storages_common_table_meta::meta::TableMetaTimestamps; use enum_as_inner::EnumAsInner; -use crate::executor::physical_plan::{PhysicalPlanDeriveHandle, PhysicalPlan}; +use crate::executor::physical_plan::PhysicalPlan; +use crate::executor::physical_plan::PhysicalPlanDeriveHandle; +use crate::executor::IPhysicalPlan; +use crate::executor::PhysicalPlanMeta; use crate::plans::CopyIntoTableMode; use crate::plans::ValidationMode; use crate::ColumnBinding; -use crate::executor::{IPhysicalPlan, PhysicalPlanMeta}; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct CopyIntoTable { @@ -58,30 +61,52 @@ impl IPhysicalPlan for CopyIntoTable { Ok(DataSchemaRefExt::create(vec![])) } - fn children<'a>(&'a self) -> Box> + 'a> { + fn children<'a>(&'a self) -> Box> + 'a> { match &self.source { CopyIntoTableSource::Query(v) => Box::new(std::iter::once(v)), CopyIntoTableSource::Stage(v) => Box::new(std::iter::once(v)), } } - fn children_mut<'a>(&'a mut self) -> Box> + 'a> { + fn children_mut<'a>( + &'a mut self, + ) -> Box> + 'a> { match &mut self.source { CopyIntoTableSource::Query(v) => Box::new(std::iter::once(v)), CopyIntoTableSource::Stage(v) => Box::new(std::iter::once(v)), } } - fn derive_with(&self, handle: &mut Box) -> Box { - let derive_input = self.input.derive_with(handle); - - match handle.derive(self, vec![derive_input]) { - Ok(v) => v, - Err(children) => { - let mut new_copy_into_table = self.clone(); - assert_eq!(children.len(), 1); - new_copy_into_table.input = children[0]; - Box::new(new_copy_into_table) + fn derive_with( + &self, + handle: &mut Box, + ) -> Box { + match &self.source { + CopyIntoTableSource::Query(query_physical_plan) => { + let derive_input = query_physical_plan.derive_with(handle); + match handle.derive(self, vec![derive_input]) { + Ok(v) => v, + Err(children) => { + let mut new_copy_into_table = self.clone(); + assert_eq!(children.len(), 1); + new_copy_into_table.source = + CopyIntoTableSource::Query(Box::new(children[0])); + Box::new(new_copy_into_table) + } + } + } + CopyIntoTableSource::Stage(v) => { + let derive_input = v.derive_with(handle); + match handle.derive(self, vec![derive_input]) { + Ok(v) => v, + Err(children) => { + let mut new_copy_into_table = self.clone(); + assert_eq!(children.len(), 1); + new_copy_into_table.source = + CopyIntoTableSource::Stage(Box::new(children[0])); + Box::new(new_copy_into_table) + } + } } } } diff --git a/src/query/sql/src/executor/physical_plans/physical_distributed_insert_select.rs b/src/query/sql/src/executor/physical_plans/physical_distributed_insert_select.rs index 85cc2ef1ed378..aa4487eed9255 100644 --- a/src/query/sql/src/executor/physical_plans/physical_distributed_insert_select.rs +++ b/src/query/sql/src/executor/physical_plans/physical_distributed_insert_select.rs @@ -13,14 +13,16 @@ // limitations under the License. use databend_common_catalog::plan::DataSourcePlan; +use databend_common_exception::Result; use databend_common_expression::DataSchemaRef; use databend_common_meta_app::schema::TableInfo; use databend_storages_common_table_meta::meta::TableMetaTimestamps; -use crate::executor::{IPhysicalPlan, PhysicalPlan, PhysicalPlanMeta}; -use crate::ColumnBinding; -use databend_common_exception::Result; use crate::executor::physical_plan::PhysicalPlanDeriveHandle; +use crate::executor::IPhysicalPlan; +use crate::executor::PhysicalPlan; +use crate::executor::PhysicalPlanMeta; +use crate::ColumnBinding; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct DistributedInsertSelect { @@ -48,11 +50,13 @@ impl IPhysicalPlan for DistributedInsertSelect { Ok(DataSchemaRef::default()) } - fn children<'a>(&'a self) -> Box> + 'a> { + fn children<'a>(&'a self) -> Box> + 'a> { Box::new(std::iter::once(&self.input)) } - fn children_mut<'a>(&'a mut self) -> Box> + 'a> { + fn children_mut<'a>( + &'a mut self, + ) -> Box> + 'a> { Box::new(std::iter::once(&mut self.input)) } @@ -61,7 +65,10 @@ impl IPhysicalPlan for DistributedInsertSelect { self.input.try_find_single_data_source() } - fn derive_with(&self, handle: &mut Box) -> Box { + fn derive_with( + &self, + handle: &mut Box, + ) -> Box { let derive_input = self.input.derive_with(handle); match handle.derive(self, vec![derive_input]) { diff --git a/src/query/sql/src/executor/physical_plans/physical_eval_scalar.rs b/src/query/sql/src/executor/physical_plans/physical_eval_scalar.rs index 24e00eae6f520..ff6d85db076e8 100644 --- a/src/query/sql/src/executor/physical_plans/physical_eval_scalar.rs +++ b/src/query/sql/src/executor/physical_plans/physical_eval_scalar.rs @@ -12,10 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::collections::{BTreeSet, HashMap}; +use std::collections::BTreeSet; +use std::collections::HashMap; use std::collections::HashSet; use std::sync::Arc; -use itertools::Itertools; + use databend_common_catalog::plan::DataSourcePlan; use databend_common_exception::Result; use databend_common_expression::ConstantFolder; @@ -25,10 +26,14 @@ use databend_common_expression::DataSchemaRefExt; use databend_common_expression::RemoteExpr; use databend_common_expression::Scalar; use databend_common_functions::BUILTIN_FUNCTIONS; +use itertools::Itertools; use crate::executor::explain::PlanStatsInfo; -use crate::executor::physical_plan::{PhysicalPlanDeriveHandle, PhysicalPlan}; +use crate::executor::physical_plan::PhysicalPlan; +use crate::executor::physical_plan::PhysicalPlanDeriveHandle; use crate::executor::physical_plan_builder::PhysicalPlanBuilder; +use crate::executor::IPhysicalPlan; +use crate::executor::PhysicalPlanMeta; use crate::optimizer::ir::Matcher; use crate::optimizer::ir::SExpr; use crate::plans::Filter; @@ -40,7 +45,6 @@ use crate::plans::ScalarExpr; use crate::plans::ScalarItem; use crate::plans::Visitor; use crate::ColumnSet; -use crate::executor::{IPhysicalPlan, PhysicalPlanMeta}; use crate::IndexType; use crate::TypeCheck; @@ -89,11 +93,13 @@ impl IPhysicalPlan for EvalScalar { Ok(DataSchemaRefExt::create(fields)) } - fn children<'a>(&'a self) -> Box> + 'a> { + fn children<'a>(&'a self) -> Box> + 'a> { Box::new(std::iter::once(&self.input)) } - fn children_mut<'a>(&'a mut self) -> Box> + 'a> { + fn children_mut<'a>( + &'a mut self, + ) -> Box> + 'a> { Box::new(std::iter::once(&mut self.input)) } @@ -107,23 +113,23 @@ impl IPhysicalPlan for EvalScalar { .exprs .iter() .map(|(x, _)| x.as_expr(&BUILTIN_FUNCTIONS).sql_display()) - .join(", ") - ) + .join(", ")) } fn get_labels(&self) -> Result>> { - Ok(HashMap::from([ - ( - String::from("List of Expressions"), - self.exprs - .iter() - .map(|(x, _)| x.as_expr(&BUILTIN_FUNCTIONS).sql_display()) - .collect(), - ) - ])) + Ok(HashMap::from([( + String::from("List of Expressions"), + self.exprs + .iter() + .map(|(x, _)| x.as_expr(&BUILTIN_FUNCTIONS).sql_display()) + .collect(), + )])) } - fn derive_with(&self, handle: &mut Box) -> Box { + fn derive_with( + &self, + handle: &mut Box, + ) -> Box { let derive_input = self.input.derive_with(handle); match handle.derive(self, vec![derive_input]) { diff --git a/src/query/sql/src/executor/physical_plans/physical_exchange.rs b/src/query/sql/src/executor/physical_plans/physical_exchange.rs index 787ca2ec2709e..4ce31edd54879 100644 --- a/src/query/sql/src/executor/physical_plans/physical_exchange.rs +++ b/src/query/sql/src/executor/physical_plans/physical_exchange.rs @@ -19,12 +19,14 @@ use databend_common_expression::DataSchemaRef; use databend_common_expression::RemoteExpr; use databend_common_functions::BUILTIN_FUNCTIONS; +use crate::executor::physical_plan::PhysicalPlanDeriveHandle; use crate::executor::physical_plans::common::FragmentKind; -use crate::executor::{IPhysicalPlan, PhysicalPlan, PhysicalPlanMeta}; +use crate::executor::IPhysicalPlan; +use crate::executor::PhysicalPlan; use crate::executor::PhysicalPlanBuilder; +use crate::executor::PhysicalPlanMeta; use crate::optimizer::ir::SExpr; use crate::ColumnSet; -use crate::executor::physical_plan::PhysicalPlanDeriveHandle; use crate::TypeCheck; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] @@ -47,11 +49,11 @@ impl IPhysicalPlan for Exchange { &mut self.meta } - fn children<'a>(&'a self) -> Box> + 'a> { + fn children<'a>(&'a self) -> Box> + 'a> { Box::new(std::iter::once(&self.input)) } - fn children_mut<'a>(&'a mut self) -> Box> + 'a> { + fn children_mut<'a>(&'a mut self) -> Box> + 'a> { Box::new(std::iter::once(&mut self.input)) } @@ -64,7 +66,10 @@ impl IPhysicalPlan for Exchange { true } - fn derive_with(&self, handle: &mut Box) -> Box { + fn derive_with( + &self, + handle: &mut Box, + ) -> Box { let derive_input = self.input.derive_with(handle); match handle.derive(self, vec![derive_input]) { diff --git a/src/query/sql/src/executor/physical_plans/physical_exchange_sink.rs b/src/query/sql/src/executor/physical_plans/physical_exchange_sink.rs index afcbfdf9414ce..3006d1d2a0f4b 100644 --- a/src/query/sql/src/executor/physical_plans/physical_exchange_sink.rs +++ b/src/query/sql/src/executor/physical_plans/physical_exchange_sink.rs @@ -17,9 +17,11 @@ use databend_common_exception::Result; use databend_common_expression::DataSchemaRef; use databend_common_expression::RemoteExpr; -use crate::executor::physical_plans::common::FragmentKind; -use crate::executor::{IPhysicalPlan, PhysicalPlan, PhysicalPlanMeta}; use crate::executor::physical_plan::PhysicalPlanDeriveHandle; +use crate::executor::physical_plans::common::FragmentKind; +use crate::executor::IPhysicalPlan; +use crate::executor::PhysicalPlan; +use crate::executor::PhysicalPlanMeta; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct ExchangeSink { @@ -53,11 +55,13 @@ impl IPhysicalPlan for ExchangeSink { Ok(self.schema.clone()) } - fn children<'a>(&'a self) -> Box> + 'a> { + fn children<'a>(&'a self) -> Box> + 'a> { Box::new(std::iter::once(&self.input)) } - fn children_mut<'a>(&'a mut self) -> Box> + 'a> { + fn children_mut<'a>( + &'a mut self, + ) -> Box> + 'a> { Box::new(std::iter::once(&mut self.input)) } @@ -70,7 +74,10 @@ impl IPhysicalPlan for ExchangeSink { true } - fn derive_with(&self, handle: &mut Box) -> Box { + fn derive_with( + &self, + handle: &mut Box, + ) -> Box { let derive_input = self.input.derive_with(handle); match handle.derive(self, vec![derive_input]) { diff --git a/src/query/sql/src/executor/physical_plans/physical_exchange_source.rs b/src/query/sql/src/executor/physical_plans/physical_exchange_source.rs index 778cb3322c7f9..b0f1f94e7ec52 100644 --- a/src/query/sql/src/executor/physical_plans/physical_exchange_source.rs +++ b/src/query/sql/src/executor/physical_plans/physical_exchange_source.rs @@ -15,8 +15,10 @@ use databend_common_catalog::plan::DataSourcePlan; use databend_common_exception::Result; use databend_common_expression::DataSchemaRef; -use crate::executor::{IPhysicalPlan, PhysicalPlanMeta}; + use crate::executor::physical_plan::PhysicalPlanDeriveHandle; +use crate::executor::IPhysicalPlan; +use crate::executor::PhysicalPlanMeta; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct ExchangeSource { @@ -48,7 +50,10 @@ impl IPhysicalPlan for ExchangeSource { true } - fn derive_with(&self, handle: &mut Box) -> Box { + fn derive_with( + &self, + handle: &mut Box, + ) -> Box { match handle.derive(self, vec![]) { Ok(v) => v, Err(children) => { diff --git a/src/query/sql/src/executor/physical_plans/physical_expression_scan.rs b/src/query/sql/src/executor/physical_plans/physical_expression_scan.rs index 6c837c9facc6e..60ad789e678fc 100644 --- a/src/query/sql/src/executor/physical_plans/physical_expression_scan.rs +++ b/src/query/sql/src/executor/physical_plans/physical_expression_scan.rs @@ -18,11 +18,13 @@ use databend_common_expression::DataSchemaRef; use databend_common_expression::RemoteExpr; use databend_common_functions::BUILTIN_FUNCTIONS; -use crate::executor::{IPhysicalPlan, PhysicalPlan, PhysicalPlanMeta}; +use crate::executor::physical_plan::PhysicalPlanDeriveHandle; +use crate::executor::IPhysicalPlan; +use crate::executor::PhysicalPlan; use crate::executor::PhysicalPlanBuilder; +use crate::executor::PhysicalPlanMeta; use crate::optimizer::ir::SExpr; use crate::ColumnSet; -use crate::executor::physical_plan::PhysicalPlanDeriveHandle; use crate::TypeCheck; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] @@ -47,15 +49,20 @@ impl IPhysicalPlan for ExpressionScan { Ok(self.output_schema.clone()) } - fn children<'a>(&'a self) -> Box> + 'a> { + fn children<'a>(&'a self) -> Box> + 'a> { Box::new(std::iter::once(&self.input)) } - fn children_mut<'a>(&'a mut self) -> Box> + 'a> { + fn children_mut<'a>( + &'a mut self, + ) -> Box> + 'a> { Box::new(std::iter::once(&mut self.input)) } - fn derive_with(&self, handle: &mut Box) -> Box { + fn derive_with( + &self, + handle: &mut Box, + ) -> Box { let derive_input = self.input.derive_with(handle); match handle.derive(self, vec![derive_input]) { diff --git a/src/query/sql/src/executor/physical_plans/physical_filter.rs b/src/query/sql/src/executor/physical_plans/physical_filter.rs index a001ffdeada4b..fa8d87a66bef6 100644 --- a/src/query/sql/src/executor/physical_plans/physical_filter.rs +++ b/src/query/sql/src/executor/physical_plans/physical_filter.rs @@ -13,6 +13,7 @@ // limitations under the License. use std::collections::HashMap; + use databend_common_catalog::plan::DataSourcePlan; use databend_common_exception::Result; use databend_common_expression::ConstantFolder; @@ -21,13 +22,15 @@ use databend_common_expression::DataSchemaRefExt; use databend_common_expression::RemoteExpr; use databend_common_functions::BUILTIN_FUNCTIONS; -use crate::executor::{cast_expr_to_non_null_boolean, IPhysicalPlan, PhysicalPlanMeta}; +use crate::executor::cast_expr_to_non_null_boolean; use crate::executor::explain::PlanStatsInfo; +use crate::executor::physical_plan::PhysicalPlanDeriveHandle; +use crate::executor::IPhysicalPlan; use crate::executor::PhysicalPlan; use crate::executor::PhysicalPlanBuilder; +use crate::executor::PhysicalPlanMeta; use crate::optimizer::ir::SExpr; use crate::ColumnSet; -use crate::executor::physical_plan::PhysicalPlanDeriveHandle; use crate::TypeCheck; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] @@ -63,11 +66,13 @@ impl IPhysicalPlan for Filter { Ok(DataSchemaRefExt::create(fields)) } - fn children<'a>(&'a self) -> Box> + 'a> { + fn children<'a>(&'a self) -> Box> + 'a> { Box::new(std::iter::once(&self.input)) } - fn children_mut<'a>(&'a mut self) -> Box> + 'a> { + fn children_mut<'a>( + &'a mut self, + ) -> Box> + 'a> { Box::new(std::iter::once(&mut self.input)) } @@ -84,18 +89,19 @@ impl IPhysicalPlan for Filter { } fn get_labels(&self) -> Result>> { - Ok(HashMap::from([ - ( - String::from("Filter condition"), - self.predicates - .iter() - .map(|x| x.as_expr(&BUILTIN_FUNCTIONS).sql_display()) - .collect(), - ) - ])) + Ok(HashMap::from([( + String::from("Filter condition"), + self.predicates + .iter() + .map(|x| x.as_expr(&BUILTIN_FUNCTIONS).sql_display()) + .collect(), + )])) } - fn derive_with(&self, handle: &mut Box) -> Box { + fn derive_with( + &self, + handle: &mut Box, + ) -> Box { let derive_input = self.input.derive_with(handle); match handle.derive(self, vec![derive_input]) { diff --git a/src/query/sql/src/executor/physical_plans/physical_hash_join.rs b/src/query/sql/src/executor/physical_plans/physical_hash_join.rs index a4698d5ad64b6..7be4725bd78dc 100644 --- a/src/query/sql/src/executor/physical_plans/physical_hash_join.rs +++ b/src/query/sql/src/executor/physical_plans/physical_hash_join.rs @@ -30,15 +30,17 @@ use databend_common_functions::BUILTIN_FUNCTIONS; use super::physical_join_filter::PhysicalRuntimeFilters; use super::JoinRuntimeFilter; use crate::executor::explain::PlanStatsInfo; +use crate::executor::physical_plan::PhysicalPlanDeriveHandle; use crate::executor::physical_plans::Exchange; -use crate::executor::{IPhysicalPlan, PhysicalPlan, PhysicalPlanMeta}; +use crate::executor::IPhysicalPlan; +use crate::executor::PhysicalPlan; use crate::executor::PhysicalPlanBuilder; +use crate::executor::PhysicalPlanMeta; use crate::optimizer::ir::SExpr; use crate::plans::Join; use crate::plans::JoinType; use crate::ColumnEntry; use crate::ColumnSet; -use crate::executor::physical_plan::PhysicalPlanDeriveHandle; use crate::IndexType; use crate::ScalarExpr; use crate::TypeCheck; @@ -122,11 +124,13 @@ impl IPhysicalPlan for HashJoin { Ok(self.output_schema.clone()) } - fn children<'a>(&'a self) -> Box> + 'a> { + fn children<'a>(&'a self) -> Box> + 'a> { Box::new(std::iter::once(&self.probe).chain(std::iter::once(&self.build))) } - fn children_mut<'a>(&'a mut self) -> Box> + 'a> { + fn children_mut<'a>( + &'a mut self, + ) -> Box> + 'a> { Box::new(std::iter::once(&mut self.probe).chain(std::iter::once(&mut self.build))) } @@ -190,7 +194,10 @@ impl IPhysicalPlan for HashJoin { Ok(labels) } - fn derive_with(&self, handle: &mut Box) -> Box { + fn derive_with( + &self, + handle: &mut Box, + ) -> Box { let derive_probe = self.probe.derive_with(handle); let derive_build = self.build.derive_with(handle); @@ -318,11 +325,11 @@ impl PhysicalPlanBuilder { // Unify the data types of the left and right exchange keys if let ( PhysicalPlan::Exchange(Exchange { - keys: probe_keys, .. - }), + keys: probe_keys, .. + }), PhysicalPlan::Exchange(Exchange { - keys: build_keys, .. - }), + keys: build_keys, .. + }), ) = (probe_side.as_mut(), build_side.as_mut()) { let cast_rules = &BUILTIN_FUNCTIONS.get_auto_cast_rules("eq"); @@ -334,12 +341,12 @@ impl PhysicalPlanBuilder { build_expr.data_type().clone(), cast_rules, ) - .ok_or_else(|| { - ErrorCode::IllegalDataType(format!( - "Cannot find common type for probe key {:?} and build key {:?}", - &probe_expr, &build_expr - )) - })?; + .ok_or_else(|| { + ErrorCode::IllegalDataType(format!( + "Cannot find common type for probe key {:?} and build key {:?}", + &probe_expr, &build_expr + )) + })?; *probe_key = check_cast( probe_expr.span(), false, @@ -347,7 +354,7 @@ impl PhysicalPlanBuilder { &common_ty, &BUILTIN_FUNCTIONS, )? - .as_remote_expr(); + .as_remote_expr(); *build_key = check_cast( build_expr.span(), false, @@ -355,7 +362,7 @@ impl PhysicalPlanBuilder { &common_ty, &BUILTIN_FUNCTIONS, )? - .as_remote_expr(); + .as_remote_expr(); } } @@ -444,9 +451,9 @@ impl PhysicalPlanBuilder { .data_type() .remove_nullable() == build_schema - .field(build_index) - .data_type() - .remove_nullable() + .field(build_index) + .data_type() + .remove_nullable() { probe_to_build_index.push(((probe_index, false), build_index)); if !pre_column_projections.contains(&left.column.index) { @@ -1049,6 +1056,6 @@ impl PhysicalPlanBuilder { build_keys, probe_keys, ) - .await + .await } } diff --git a/src/query/sql/src/executor/physical_plans/physical_join.rs b/src/query/sql/src/executor/physical_plans/physical_join.rs index 573c7eb139665..e19e0ba868e1b 100644 --- a/src/query/sql/src/executor/physical_plans/physical_join.rs +++ b/src/query/sql/src/executor/physical_plans/physical_join.rs @@ -16,7 +16,8 @@ use databend_common_exception::Result; use crate::binder::JoinPredicate; use crate::executor::explain::PlanStatsInfo; -use crate::executor::{IPhysicalPlan, PhysicalPlan}; +use crate::executor::IPhysicalPlan; +use crate::executor::PhysicalPlan; use crate::executor::PhysicalPlanBuilder; use crate::optimizer::ir::RelExpr; use crate::optimizer::ir::RelationalProperty; diff --git a/src/query/sql/src/executor/physical_plans/physical_limit.rs b/src/query/sql/src/executor/physical_plans/physical_limit.rs index 9bc375972725e..e9655c2917e71 100644 --- a/src/query/sql/src/executor/physical_plans/physical_limit.rs +++ b/src/query/sql/src/executor/physical_plans/physical_limit.rs @@ -13,6 +13,7 @@ // limitations under the License. use std::collections::HashMap; + use databend_common_catalog::plan::DataSourcePlan; use databend_common_exception::ErrorCode; use databend_common_exception::Result; @@ -21,13 +22,15 @@ use databend_common_expression::DataSchemaRef; use databend_common_expression::ROW_ID_COL_NAME; use crate::executor::explain::PlanStatsInfo; +use crate::executor::physical_plan::PhysicalPlanDeriveHandle; use crate::executor::physical_plans::physical_row_fetch::RowFetch; -use crate::executor::{IPhysicalPlan, PhysicalPlan, PhysicalPlanMeta}; +use crate::executor::IPhysicalPlan; +use crate::executor::PhysicalPlan; use crate::executor::PhysicalPlanBuilder; +use crate::executor::PhysicalPlanMeta; use crate::optimizer::ir::SExpr; use crate::ColumnEntry; use crate::ColumnSet; -use crate::executor::physical_plan::PhysicalPlanDeriveHandle; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct Limit { @@ -50,11 +53,13 @@ impl IPhysicalPlan for Limit { &mut self.meta } - fn children<'a>(&'a self) -> Box> + 'a> { + fn children<'a>(&'a self) -> Box> + 'a> { Box::new(std::iter::once(&self.input)) } - fn children_mut<'a>(&'a mut self) -> Box> + 'a> { + fn children_mut<'a>( + &'a mut self, + ) -> Box> + 'a> { Box::new(std::iter::once(&mut self.input)) } @@ -81,7 +86,10 @@ impl IPhysicalPlan for Limit { Ok(labels) } - fn derive_with(&self, handle: &mut Box) -> Box { + fn derive_with( + &self, + handle: &mut Box, + ) -> Box { let derive_input = self.input.derive_with(handle); match handle.derive(self, vec![derive_input]) { diff --git a/src/query/sql/src/executor/physical_plans/physical_multi_table_insert.rs b/src/query/sql/src/executor/physical_plans/physical_multi_table_insert.rs index dc90adf8c0e60..17dc0b004128f 100644 --- a/src/query/sql/src/executor/physical_plans/physical_multi_table_insert.rs +++ b/src/query/sql/src/executor/physical_plans/physical_multi_table_insert.rs @@ -23,9 +23,11 @@ use databend_common_meta_app::schema::TableInfo; use databend_common_meta_app::schema::UpdateStreamMetaReq; use databend_storages_common_table_meta::meta::TableMetaTimestamps; -use crate::executor::{IPhysicalPlan, PhysicalPlan, PhysicalPlanMeta}; -use crate::ColumnSet; use crate::executor::physical_plan::PhysicalPlanDeriveHandle; +use crate::executor::IPhysicalPlan; +use crate::executor::PhysicalPlan; +use crate::executor::PhysicalPlanMeta; +use crate::ColumnSet; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct Duplicate { @@ -44,15 +46,20 @@ impl IPhysicalPlan for Duplicate { &mut self.meta } - fn children<'a>(&'a self) -> Box> + 'a> { + fn children<'a>(&'a self) -> Box> + 'a> { Box::new(std::iter::once(&self.input)) } - fn children_mut<'a>(&'a mut self) -> Box> + 'a> { + fn children_mut<'a>( + &'a mut self, + ) -> Box> + 'a> { Box::new(std::iter::once(&mut self.input)) } - fn derive_with(&self, handle: &mut Box) -> Box { + fn derive_with( + &self, + handle: &mut Box, + ) -> Box { let derive_input = self.input.derive_with(handle); match handle.derive(self, vec![derive_input]) { @@ -84,15 +91,20 @@ impl IPhysicalPlan for Shuffle { &mut self.meta } - fn children<'a>(&'a self) -> Box> + 'a> { + fn children<'a>(&'a self) -> Box> + 'a> { Box::new(std::iter::once(&self.input)) } - fn children_mut<'a>(&'a mut self) -> Box> + 'a> { + fn children_mut<'a>( + &'a mut self, + ) -> Box> + 'a> { Box::new(std::iter::once(&mut self.input)) } - fn derive_with(&self, handle: &mut Box) -> Box { + fn derive_with( + &self, + handle: &mut Box, + ) -> Box { let derive_input = self.input.derive_with(handle); match handle.derive(self, vec![derive_input]) { @@ -151,15 +163,20 @@ impl IPhysicalPlan for ChunkFilter { &mut self.meta } - fn children<'a>(&'a self) -> Box> + 'a> { + fn children<'a>(&'a self) -> Box> + 'a> { Box::new(std::iter::once(&self.input)) } - fn children_mut<'a>(&'a mut self) -> Box> + 'a> { + fn children_mut<'a>( + &'a mut self, + ) -> Box> + 'a> { Box::new(std::iter::once(&mut self.input)) } - fn derive_with(&self, handle: &mut Box) -> Box { + fn derive_with( + &self, + handle: &mut Box, + ) -> Box { let derive_input = self.input.derive_with(handle); match handle.derive(self, vec![derive_input]) { @@ -191,15 +208,20 @@ impl IPhysicalPlan for ChunkEvalScalar { &mut self.meta } - fn children<'a>(&'a self) -> Box> + 'a> { + fn children<'a>(&'a self) -> Box> + 'a> { Box::new(std::iter::once(&self.input)) } - fn children_mut<'a>(&'a mut self) -> Box> + 'a> { + fn children_mut<'a>( + &'a mut self, + ) -> Box> + 'a> { Box::new(std::iter::once(&mut self.input)) } - fn derive_with(&self, handle: &mut Box) -> Box { + fn derive_with( + &self, + handle: &mut Box, + ) -> Box { let derive_input = self.input.derive_with(handle); match handle.derive(self, vec![derive_input]) { @@ -237,15 +259,20 @@ impl IPhysicalPlan for ChunkCastSchema { &mut self.meta } - fn children<'a>(&'a self) -> Box> + 'a> { + fn children<'a>(&'a self) -> Box> + 'a> { Box::new(std::iter::once(&self.input)) } - fn children_mut<'a>(&'a mut self) -> Box> + 'a> { + fn children_mut<'a>( + &'a mut self, + ) -> Box> + 'a> { Box::new(std::iter::once(&mut self.input)) } - fn derive_with(&self, handle: &mut Box) -> Box { + fn derive_with( + &self, + handle: &mut Box, + ) -> Box { let derive_input = self.input.derive_with(handle); match handle.derive(self, vec![derive_input]) { @@ -283,15 +310,20 @@ impl IPhysicalPlan for ChunkFillAndReorder { &mut self.meta } - fn children<'a>(&'a self) -> Box> + 'a> { + fn children<'a>(&'a self) -> Box> + 'a> { Box::new(std::iter::once(&self.input)) } - fn children_mut<'a>(&'a mut self) -> Box> + 'a> { + fn children_mut<'a>( + &'a mut self, + ) -> Box> + 'a> { Box::new(std::iter::once(&mut self.input)) } - fn derive_with(&self, handle: &mut Box) -> Box { + fn derive_with( + &self, + handle: &mut Box, + ) -> Box { let derive_input = self.input.derive_with(handle); match handle.derive(self, vec![derive_input]) { @@ -329,15 +361,20 @@ impl IPhysicalPlan for ChunkAppendData { &mut self.meta } - fn children<'a>(&'a self) -> Box> + 'a> { + fn children<'a>(&'a self) -> Box> + 'a> { Box::new(std::iter::once(&self.input)) } - fn children_mut<'a>(&'a mut self) -> Box> + 'a> { + fn children_mut<'a>( + &'a mut self, + ) -> Box> + 'a> { Box::new(std::iter::once(&mut self.input)) } - fn derive_with(&self, handle: &mut Box) -> Box { + fn derive_with( + &self, + handle: &mut Box, + ) -> Box { let derive_input = self.input.derive_with(handle); match handle.derive(self, vec![derive_input]) { @@ -376,15 +413,20 @@ impl IPhysicalPlan for ChunkMerge { &mut self.meta } - fn children<'a>(&'a self) -> Box> + 'a> { + fn children<'a>(&'a self) -> Box> + 'a> { Box::new(std::iter::once(&self.input)) } - fn children_mut<'a>(&'a mut self) -> Box> + 'a> { + fn children_mut<'a>( + &'a mut self, + ) -> Box> + 'a> { Box::new(std::iter::once(&mut self.input)) } - fn derive_with(&self, handle: &mut Box) -> Box { + fn derive_with( + &self, + handle: &mut Box, + ) -> Box { let derive_input = self.input.derive_with(handle); match handle.derive(self, vec![derive_input]) { @@ -419,15 +461,20 @@ impl IPhysicalPlan for ChunkCommitInsert { &mut self.meta } - fn children<'a>(&'a self) -> Box> + 'a> { + fn children<'a>(&'a self) -> Box> + 'a> { Box::new(std::iter::once(&self.input)) } - fn children_mut<'a>(&'a mut self) -> Box> + 'a> { + fn children_mut<'a>( + &'a mut self, + ) -> Box> + 'a> { Box::new(std::iter::once(&mut self.input)) } - fn derive_with(&self, handle: &mut Box) -> Box { + fn derive_with( + &self, + handle: &mut Box, + ) -> Box { let derive_input = self.input.derive_with(handle); match handle.derive(self, vec![derive_input]) { diff --git a/src/query/sql/src/executor/physical_plans/physical_mutation.rs b/src/query/sql/src/executor/physical_plans/physical_mutation.rs index e3b865159e352..48a2aca6959f1 100644 --- a/src/query/sql/src/executor/physical_plans/physical_mutation.rs +++ b/src/query/sql/src/executor/physical_plans/physical_mutation.rs @@ -15,7 +15,8 @@ use std::collections::HashMap; use std::sync::Arc; -use databend_common_catalog::plan::{DataSourcePlan, NUM_ROW_ID_PREFIX_BITS}; +use databend_common_catalog::plan::DataSourcePlan; +use databend_common_catalog::plan::NUM_ROW_ID_PREFIX_BITS; use databend_common_catalog::table_context::TableContext; use databend_common_exception::ErrorCode; use databend_common_exception::Result; @@ -48,7 +49,8 @@ use super::CommitType; use crate::binder::wrap_cast; use crate::binder::MutationStrategy; use crate::binder::MutationType; -use crate::executor::physical_plan::{PhysicalPlanDeriveHandle, PhysicalPlan}; +use crate::executor::physical_plan::PhysicalPlan; +use crate::executor::physical_plan::PhysicalPlanDeriveHandle; use crate::executor::physical_plans::CommitSink; use crate::executor::physical_plans::Exchange; use crate::executor::physical_plans::FragmentKind; @@ -57,7 +59,9 @@ use crate::executor::physical_plans::MutationManipulate; use crate::executor::physical_plans::MutationOrganize; use crate::executor::physical_plans::MutationSplit; use crate::executor::physical_plans::RowFetch; -use crate::executor::{IPhysicalPlan, PhysicalPlanBuilder, PhysicalPlanMeta}; +use crate::executor::IPhysicalPlan; +use crate::executor::PhysicalPlanBuilder; +use crate::executor::PhysicalPlanMeta; use crate::optimizer::ir::SExpr; use crate::parse_computed_expr; use crate::plans::BoundColumnRef; @@ -109,15 +113,20 @@ impl IPhysicalPlan for Mutation { Ok(DataSchemaRef::default()) } - fn children<'a>(&'a self) -> Box> + 'a> { + fn children<'a>(&'a self) -> Box> + 'a> { Box::new(std::iter::once(&self.input)) } - fn children_mut<'a>(&'a mut self) -> Box> + 'a> { + fn children_mut<'a>( + &'a mut self, + ) -> Box> + 'a> { Box::new(std::iter::once(&mut self.input)) } - fn derive_with(&self, handle: &mut Box) -> Box { + fn derive_with( + &self, + handle: &mut Box, + ) -> Box { let derive_input = self.input.derive_with(handle); match handle.derive(self, vec![derive_input]) { @@ -686,7 +695,7 @@ pub fn generate_update_list( Box::new(DataType::Boolean), Visibility::Visible, ) - .build(); + .build(); let predicate = ScalarExpr::BoundColumnRef(BoundColumnRef { span: None, column }); update_list.iter().try_fold( @@ -771,7 +780,7 @@ pub fn mutation_update_expr( Box::new(DataType::Boolean), Visibility::Visible, ) - .build(); + .build(); ScalarExpr::BoundColumnRef(BoundColumnRef { span: None, column }) } else { ScalarExpr::ConstantExpr(ConstantExpr { diff --git a/src/query/sql/src/executor/physical_plans/physical_mutation_into_organize.rs b/src/query/sql/src/executor/physical_plans/physical_mutation_into_organize.rs index 8feccff28c7a6..d70d739457159 100644 --- a/src/query/sql/src/executor/physical_plans/physical_mutation_into_organize.rs +++ b/src/query/sql/src/executor/physical_plans/physical_mutation_into_organize.rs @@ -16,8 +16,10 @@ use databend_common_exception::Result; use databend_common_expression::DataSchemaRef; use crate::binder::MutationStrategy; -use crate::executor::{IPhysicalPlan, PhysicalPlanMeta}; -use crate::executor::physical_plan::{PhysicalPlanDeriveHandle, PhysicalPlan}; +use crate::executor::physical_plan::PhysicalPlan; +use crate::executor::physical_plan::PhysicalPlanDeriveHandle; +use crate::executor::IPhysicalPlan; +use crate::executor::PhysicalPlanMeta; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct MutationOrganize { @@ -36,15 +38,20 @@ impl IPhysicalPlan for MutationOrganize { &mut self.meta } - fn children<'a>(&'a self) -> Box> + 'a> { + fn children<'a>(&'a self) -> Box> + 'a> { Box::new(std::iter::once(&self.input)) } - fn children_mut<'a>(&'a mut self) -> Box> + 'a> { + fn children_mut<'a>( + &'a mut self, + ) -> Box> + 'a> { Box::new(std::iter::once(&mut self.input)) } - fn derive_with(&self, handle: &mut Box) -> Box { + fn derive_with( + &self, + handle: &mut Box, + ) -> Box { let derive_input = self.input.derive_with(handle); match handle.derive(self, vec![derive_input]) { diff --git a/src/query/sql/src/executor/physical_plans/physical_mutation_into_split.rs b/src/query/sql/src/executor/physical_plans/physical_mutation_into_split.rs index 18bb39a19d412..86b1352ed6c07 100644 --- a/src/query/sql/src/executor/physical_plans/physical_mutation_into_split.rs +++ b/src/query/sql/src/executor/physical_plans/physical_mutation_into_split.rs @@ -14,8 +14,11 @@ use databend_common_exception::Result; use databend_common_expression::DataSchemaRef; -use crate::executor::{IPhysicalPlan, PhysicalPlanMeta}; -use crate::executor::physical_plan::{PhysicalPlanDeriveHandle, PhysicalPlan}; + +use crate::executor::physical_plan::PhysicalPlan; +use crate::executor::physical_plan::PhysicalPlanDeriveHandle; +use crate::executor::IPhysicalPlan; +use crate::executor::PhysicalPlanMeta; use crate::IndexType; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] @@ -35,15 +38,20 @@ impl IPhysicalPlan for MutationSplit { &mut self.meta } - fn children<'a>(&'a self) -> Box> + 'a> { + fn children<'a>(&'a self) -> Box> + 'a> { Box::new(std::iter::once(&self.input)) } - fn children_mut<'a>(&'a mut self) -> Box> + 'a> { + fn children_mut<'a>( + &'a mut self, + ) -> Box> + 'a> { Box::new(std::iter::once(&mut self.input)) } - fn derive_with(&self, handle: &mut Box) -> Box { + fn derive_with( + &self, + handle: &mut Box, + ) -> Box { let derive_input = self.input.derive_with(handle); match handle.derive(self, vec![derive_input]) { diff --git a/src/query/sql/src/executor/physical_plans/physical_mutation_manipulate.rs b/src/query/sql/src/executor/physical_plans/physical_mutation_manipulate.rs index 225620af61cec..6cc2de33b9fa8 100644 --- a/src/query/sql/src/executor/physical_plans/physical_mutation_manipulate.rs +++ b/src/query/sql/src/executor/physical_plans/physical_mutation_manipulate.rs @@ -21,8 +21,10 @@ use databend_common_expression::RemoteExpr; use databend_common_meta_app::schema::TableInfo; use crate::binder::MutationStrategy; -use crate::executor::{IPhysicalPlan, PhysicalPlanMeta}; -use crate::executor::physical_plan::{PhysicalPlanDeriveHandle, PhysicalPlan}; +use crate::executor::physical_plan::PhysicalPlan; +use crate::executor::physical_plan::PhysicalPlanDeriveHandle; +use crate::executor::IPhysicalPlan; +use crate::executor::PhysicalPlanMeta; pub type MatchExpr = Vec<(Option, Option>)>; @@ -54,15 +56,20 @@ impl IPhysicalPlan for MutationManipulate { &mut self.meta } - fn children<'a>(&'a self) -> Box> + 'a> { + fn children<'a>(&'a self) -> Box> + 'a> { Box::new(std::iter::once(&self.input)) } - fn children_mut<'a>(&'a mut self) -> Box> + 'a> { + fn children_mut<'a>( + &'a mut self, + ) -> Box> + 'a> { Box::new(std::iter::once(&mut self.input)) } - fn derive_with(&self, handle: &mut Box) -> Box { + fn derive_with( + &self, + handle: &mut Box, + ) -> Box { let derive_input = self.input.derive_with(handle); match handle.derive(self, vec![derive_input]) { diff --git a/src/query/sql/src/executor/physical_plans/physical_mutation_source.rs b/src/query/sql/src/executor/physical_plans/physical_mutation_source.rs index 65c8827f3ea59..1faa7276c4f3a 100644 --- a/src/query/sql/src/executor/physical_plans/physical_mutation_source.rs +++ b/src/query/sql/src/executor/physical_plans/physical_mutation_source.rs @@ -12,7 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -use databend_common_catalog::plan::{DataSourcePlan, Filters}; +use databend_common_catalog::plan::DataSourcePlan; +use databend_common_catalog::plan::Filters; use databend_common_catalog::plan::PartStatistics; use databend_common_catalog::plan::Partitions; use databend_common_exception::Result; @@ -27,11 +28,13 @@ use databend_common_functions::BUILTIN_FUNCTIONS; use databend_common_meta_app::schema::TableInfo; use crate::binder::MutationType; -use crate::executor::{cast_expr_to_non_null_boolean, IPhysicalPlan, PhysicalPlanMeta}; +use crate::executor::cast_expr_to_non_null_boolean; +use crate::executor::physical_plan::PhysicalPlanDeriveHandle; +use crate::executor::IPhysicalPlan; use crate::executor::PhysicalPlan; use crate::executor::PhysicalPlanBuilder; +use crate::executor::PhysicalPlanMeta; use crate::ColumnSet; -use crate::executor::physical_plan::PhysicalPlanDeriveHandle; use crate::IndexType; use crate::ScalarExpr; @@ -68,7 +71,10 @@ impl IPhysicalPlan for MutationSource { Some(self.clone()) } - fn derive_with(&self, handle: &mut Box) -> Box { + fn derive_with( + &self, + handle: &mut Box, + ) -> Box { match handle.derive(self, vec![]) { Ok(v) => v, Err(children) => { diff --git a/src/query/sql/src/executor/physical_plans/physical_project_set.rs b/src/query/sql/src/executor/physical_plans/physical_project_set.rs index ae1fcc48b6c73..dd20b5d91c8c9 100644 --- a/src/query/sql/src/executor/physical_plans/physical_project_set.rs +++ b/src/query/sql/src/executor/physical_plans/physical_project_set.rs @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -use itertools::Itertools; use databend_common_catalog::plan::DataSourcePlan; use databend_common_exception::Result; use databend_common_expression::ConstantFolder; @@ -21,13 +20,16 @@ use databend_common_expression::DataSchemaRef; use databend_common_expression::DataSchemaRefExt; use databend_common_expression::RemoteExpr; use databend_common_functions::BUILTIN_FUNCTIONS; +use itertools::Itertools; use crate::executor::explain::PlanStatsInfo; -use crate::executor::{IPhysicalPlan, PhysicalPlan, PhysicalPlanMeta}; +use crate::executor::physical_plan::PhysicalPlanDeriveHandle; +use crate::executor::IPhysicalPlan; +use crate::executor::PhysicalPlan; use crate::executor::PhysicalPlanBuilder; +use crate::executor::PhysicalPlanMeta; use crate::optimizer::ir::SExpr; use crate::ColumnSet; -use crate::executor::physical_plan::PhysicalPlanDeriveHandle; use crate::IndexType; use crate::TypeCheck; @@ -69,11 +71,13 @@ impl IPhysicalPlan for ProjectSet { Ok(DataSchemaRefExt::create(fields)) } - fn children<'a>(&'a self) -> Box> + 'a> { + fn children<'a>(&'a self) -> Box> + 'a> { Box::new(std::iter::once(&self.input)) } - fn children_mut<'a>(&'a mut self) -> Box> + 'a> { + fn children_mut<'a>( + &'a mut self, + ) -> Box> + 'a> { Box::new(std::iter::once(&mut self.input)) } @@ -90,7 +94,10 @@ impl IPhysicalPlan for ProjectSet { .join(", ")) } - fn derive_with(&self, handle: &mut Box) -> Box { + fn derive_with( + &self, + handle: &mut Box, + ) -> Box { let derive_input = self.input.derive_with(handle); match handle.derive(self, vec![derive_input]) { diff --git a/src/query/sql/src/executor/physical_plans/physical_r_cte_scan.rs b/src/query/sql/src/executor/physical_plans/physical_r_cte_scan.rs index 81d7d43ead0c3..e59e6314f95f9 100644 --- a/src/query/sql/src/executor/physical_plans/physical_r_cte_scan.rs +++ b/src/query/sql/src/executor/physical_plans/physical_r_cte_scan.rs @@ -19,9 +19,11 @@ use databend_common_expression::DataSchemaRef; use databend_common_expression::DataSchemaRefExt; use crate::executor::explain::PlanStatsInfo; -use crate::executor::{IPhysicalPlan, PhysicalPlan, PhysicalPlanMeta}; use crate::executor::physical_plan::PhysicalPlanDeriveHandle; +use crate::executor::IPhysicalPlan; +use crate::executor::PhysicalPlan; use crate::executor::PhysicalPlanBuilder; +use crate::executor::PhysicalPlanMeta; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct RecursiveCteScan { @@ -45,7 +47,10 @@ impl IPhysicalPlan for RecursiveCteScan { Ok(self.output_schema.clone()) } - fn derive_with(&self, handle: &mut Box) -> Box { + fn derive_with( + &self, + handle: &mut Box, + ) -> Box { match handle.derive(self, vec![]) { Ok(v) => v, Err(children) => { diff --git a/src/query/sql/src/executor/physical_plans/physical_range_join.rs b/src/query/sql/src/executor/physical_plans/physical_range_join.rs index a0a6b21fb83c8..b3a9c81efe54b 100644 --- a/src/query/sql/src/executor/physical_plans/physical_range_join.rs +++ b/src/query/sql/src/executor/physical_plans/physical_range_join.rs @@ -26,15 +26,17 @@ use databend_common_functions::BUILTIN_FUNCTIONS; use crate::binder::wrap_cast; use crate::binder::JoinPredicate; use crate::executor::explain::PlanStatsInfo; -use crate::executor::{IPhysicalPlan, PhysicalPlan, PhysicalPlanMeta}; +use crate::executor::physical_plan::PhysicalPlanDeriveHandle; +use crate::executor::IPhysicalPlan; +use crate::executor::PhysicalPlan; use crate::executor::PhysicalPlanBuilder; +use crate::executor::PhysicalPlanMeta; use crate::optimizer::ir::RelExpr; use crate::optimizer::ir::RelationalProperty; use crate::optimizer::ir::SExpr; use crate::plans::Join; use crate::plans::JoinType; use crate::ColumnSet; -use crate::executor::physical_plan::PhysicalPlanDeriveHandle; use crate::ScalarExpr; use crate::TypeCheck; @@ -71,11 +73,13 @@ impl IPhysicalPlan for RangeJoin { Ok(self.output_schema.clone()) } - fn children<'a>(&'a self) -> Box> + 'a> { + fn children<'a>(&'a self) -> Box> + 'a> { Box::new(std::iter::once(&self.left).chain(std::iter::once(&self.right))) } - fn children_mut<'a>(&'a mut self) -> Box> + 'a> { + fn children_mut<'a>( + &'a mut self, + ) -> Box> + 'a> { Box::new(std::iter::once(&mut self.left).chain(std::iter::once(&mut self.right))) } @@ -105,7 +109,10 @@ impl IPhysicalPlan for RangeJoin { Ok(condition.join(" AND ")) } - fn derive_with(&self, handle: &mut Box) -> Box { + fn derive_with( + &self, + handle: &mut Box, + ) -> Box { let derive_left = self.left.derive_with(handle); let derive_right = self.right.derive_with(handle); @@ -231,10 +238,10 @@ fn resolve_range_condition( let common_type = common_super_type(arg1_data_type.clone(), arg2_data_type.clone(), cast_rules) .ok_or_else(|| { - ErrorCode::IllegalDataType(format!( - "Cannot find common type for {arg1_data_type} and {arg2_data_type}" - )) - })?; + ErrorCode::IllegalDataType(format!( + "Cannot find common type for {arg1_data_type} and {arg2_data_type}" + )) + })?; arg1 = wrap_cast(&arg1, &common_type); arg2 = wrap_cast(&arg2, &common_type); }; diff --git a/src/query/sql/src/executor/physical_plans/physical_recluster.rs b/src/query/sql/src/executor/physical_plans/physical_recluster.rs index e15a00f937964..3995808abd150 100644 --- a/src/query/sql/src/executor/physical_plans/physical_recluster.rs +++ b/src/query/sql/src/executor/physical_plans/physical_recluster.rs @@ -12,14 +12,17 @@ // See the License for the specific language governing permissions and // limitations under the License. -use databend_common_catalog::plan::{DataSourcePlan, ReclusterTask}; +use databend_common_catalog::plan::DataSourcePlan; +use databend_common_catalog::plan::ReclusterTask; +use databend_common_exception::Result; use databend_common_expression::DataSchemaRef; use databend_common_meta_app::schema::TableInfo; use databend_storages_common_table_meta::meta::TableMetaTimestamps; -use crate::executor::{IPhysicalPlan, PhysicalPlan, PhysicalPlanMeta}; -use databend_common_exception::Result; use crate::executor::physical_plan::PhysicalPlanDeriveHandle; +use crate::executor::IPhysicalPlan; +use crate::executor::PhysicalPlan; +use crate::executor::PhysicalPlanMeta; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct Recluster { @@ -39,7 +42,10 @@ impl IPhysicalPlan for Recluster { &mut self.meta } - fn derive_with(&self, handle: &mut Box) -> Box { + fn derive_with( + &self, + handle: &mut Box, + ) -> Box { match handle.derive(self, vec![]) { Ok(v) => v, Err(children) => { @@ -74,15 +80,20 @@ impl IPhysicalPlan for HilbertPartition { Ok(DataSchemaRef::default()) } - fn children<'a>(&'a self) -> Box> + 'a> { + fn children<'a>(&'a self) -> Box> + 'a> { Box::new(std::iter::once(&self.input)) } - fn children_mut<'a>(&'a mut self) -> Box> + 'a> { + fn children_mut<'a>( + &'a mut self, + ) -> Box> + 'a> { Box::new(std::iter::once(&mut self.input)) } - fn derive_with(&self, handle: &mut Box) -> Box { + fn derive_with( + &self, + handle: &mut Box, + ) -> Box { let derive_input = self.input.derive_with(handle); match handle.derive(self, vec![derive_input]) { diff --git a/src/query/sql/src/executor/physical_plans/physical_replace_async_source.rs b/src/query/sql/src/executor/physical_plans/physical_replace_async_source.rs index c35d86d91a836..7e16a3ba47d5b 100644 --- a/src/query/sql/src/executor/physical_plans/physical_replace_async_source.rs +++ b/src/query/sql/src/executor/physical_plans/physical_replace_async_source.rs @@ -14,8 +14,10 @@ use databend_common_catalog::plan::DataSourcePlan; use databend_common_expression::DataSchemaRef; -use crate::executor::{IPhysicalPlan, PhysicalPlanMeta}; + use crate::executor::physical_plan::PhysicalPlanDeriveHandle; +use crate::executor::IPhysicalPlan; +use crate::executor::PhysicalPlanMeta; use crate::plans::InsertValue; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] @@ -35,7 +37,10 @@ impl IPhysicalPlan for ReplaceAsyncSourcer { &mut self.meta } - fn derive_with(&self, handle: &mut Box) -> Box { + fn derive_with( + &self, + handle: &mut Box, + ) -> Box { match handle.derive(self, vec![]) { Ok(v) => v, Err(children) => { diff --git a/src/query/sql/src/executor/physical_plans/physical_replace_deduplicate.rs b/src/query/sql/src/executor/physical_plans/physical_replace_deduplicate.rs index 7687a16da4bad..28168f0a7906e 100644 --- a/src/query/sql/src/executor/physical_plans/physical_replace_deduplicate.rs +++ b/src/query/sql/src/executor/physical_plans/physical_replace_deduplicate.rs @@ -13,7 +13,9 @@ // limitations under the License. use std::collections::HashMap; + use databend_common_catalog::plan::DataSourcePlan; +use databend_common_exception::Result; use databend_common_expression::ColumnId; use databend_common_expression::DataSchemaRef; use databend_common_expression::FieldIndex; @@ -22,11 +24,12 @@ use databend_common_expression::TableSchemaRef; use databend_common_meta_app::schema::TableInfo; use databend_storages_common_table_meta::meta::ColumnStatistics; +use crate::executor::physical_plan::PhysicalPlanDeriveHandle; use crate::executor::physical_plans::common::OnConflictField; -use crate::executor::{IPhysicalPlan, PhysicalPlan, PhysicalPlanMeta}; +use crate::executor::IPhysicalPlan; +use crate::executor::PhysicalPlan; +use crate::executor::PhysicalPlanMeta; use crate::ColumnBinding; -use databend_common_exception::Result; -use crate::executor::physical_plan::PhysicalPlanDeriveHandle; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct ReplaceDeduplicate { @@ -57,15 +60,20 @@ impl IPhysicalPlan for ReplaceDeduplicate { Ok(DataSchemaRef::default()) } - fn children<'a>(&'a self) -> Box> + 'a> { + fn children<'a>(&'a self) -> Box> + 'a> { Box::new(std::iter::once(&self.input)) } - fn children_mut<'a>(&'a mut self) -> Box> + 'a> { + fn children_mut<'a>( + &'a mut self, + ) -> Box> + 'a> { Box::new(std::iter::once(&mut self.input)) } - fn derive_with(&self, handle: &mut Box) -> Box { + fn derive_with( + &self, + handle: &mut Box, + ) -> Box { let derive_input = self.input.derive_with(handle); match handle.derive(self, vec![derive_input]) { diff --git a/src/query/sql/src/executor/physical_plans/physical_replace_into.rs b/src/query/sql/src/executor/physical_plans/physical_replace_into.rs index 891b8444727ea..566ee4e4f41f4 100644 --- a/src/query/sql/src/executor/physical_plans/physical_replace_into.rs +++ b/src/query/sql/src/executor/physical_plans/physical_replace_into.rs @@ -13,17 +13,20 @@ // limitations under the License. use databend_common_catalog::plan::DataSourcePlan; -use databend_common_expression::{BlockThresholds, DataSchemaRef}; +use databend_common_exception::Result; +use databend_common_expression::BlockThresholds; +use databend_common_expression::DataSchemaRef; use databend_common_expression::FieldIndex; use databend_common_meta_app::schema::TableInfo; use databend_storages_common_table_meta::meta::BlockSlotDescription; use databend_storages_common_table_meta::meta::Location; use databend_storages_common_table_meta::meta::TableMetaTimestamps; -use crate::executor::physical_plans::common::OnConflictField; -use crate::executor::{IPhysicalPlan, PhysicalPlan, PhysicalPlanMeta}; -use databend_common_exception::Result; use crate::executor::physical_plan::PhysicalPlanDeriveHandle; +use crate::executor::physical_plans::common::OnConflictField; +use crate::executor::IPhysicalPlan; +use crate::executor::PhysicalPlan; +use crate::executor::PhysicalPlanMeta; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct ReplaceInto { @@ -53,15 +56,20 @@ impl IPhysicalPlan for ReplaceInto { Ok(DataSchemaRef::default()) } - fn children<'a>(&'a self) -> Box> + 'a> { + fn children<'a>(&'a self) -> Box> + 'a> { Box::new(std::iter::once(&self.input)) } - fn children_mut<'a>(&'a mut self) -> Box> + 'a> { + fn children_mut<'a>( + &'a mut self, + ) -> Box> + 'a> { Box::new(std::iter::once(&mut self.input)) } - fn derive_with(&self, handle: &mut Box) -> Box { + fn derive_with( + &self, + handle: &mut Box, + ) -> Box { let derive_input = self.input.derive_with(handle); match handle.derive(self, vec![derive_input]) { diff --git a/src/query/sql/src/executor/physical_plans/physical_row_fetch.rs b/src/query/sql/src/executor/physical_plans/physical_row_fetch.rs index 96ae64e5dcc81..c2faeadc48080 100644 --- a/src/query/sql/src/executor/physical_plans/physical_row_fetch.rs +++ b/src/query/sql/src/executor/physical_plans/physical_row_fetch.rs @@ -12,17 +12,19 @@ // See the License for the specific language governing permissions and // limitations under the License. -use itertools::Itertools; use databend_common_catalog::plan::DataSourcePlan; use databend_common_catalog::plan::Projection; use databend_common_exception::Result; use databend_common_expression::DataField; use databend_common_expression::DataSchemaRef; use databend_common_expression::DataSchemaRefExt; +use itertools::Itertools; use crate::executor::explain::PlanStatsInfo; -use crate::executor::{IPhysicalPlan, PhysicalPlan, PhysicalPlanMeta}; use crate::executor::physical_plan::PhysicalPlanDeriveHandle; +use crate::executor::IPhysicalPlan; +use crate::executor::PhysicalPlan; +use crate::executor::PhysicalPlanMeta; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct RowFetch { @@ -56,11 +58,13 @@ impl IPhysicalPlan for RowFetch { Ok(DataSchemaRefExt::create(fields)) } - fn children<'a>(&'a self) -> Box> + 'a> { + fn children<'a>(&'a self) -> Box> + 'a> { Box::new(std::iter::once(&self.input)) } - fn children_mut<'a>(&'a mut self) -> Box> + 'a> { + fn children_mut<'a>( + &'a mut self, + ) -> Box> + 'a> { Box::new(std::iter::once(&mut self.input)) } @@ -75,7 +79,10 @@ impl IPhysicalPlan for RowFetch { Ok(projected_schema.fields.iter().map(|f| f.name()).join(", ")) } - fn derive_with(&self, handle: &mut Box) -> Box { + fn derive_with( + &self, + handle: &mut Box, + ) -> Box { let derive_input = self.input.derive_with(handle); match handle.derive(self, vec![derive_input]) { diff --git a/src/query/sql/src/executor/physical_plans/physical_sort.rs b/src/query/sql/src/executor/physical_plans/physical_sort.rs index 284bd2f88401c..35f194039bb7b 100644 --- a/src/query/sql/src/executor/physical_plans/physical_sort.rs +++ b/src/query/sql/src/executor/physical_plans/physical_sort.rs @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -use itertools::Itertools; use databend_common_catalog::plan::DataSourcePlan; use databend_common_exception::Result; use databend_common_expression::types::DataType; @@ -21,18 +20,21 @@ use databend_common_expression::DataSchema; use databend_common_expression::DataSchemaRef; use databend_common_expression::DataSchemaRefExt; use databend_common_pipeline_transforms::processors::sort::utils::ORDER_COL_NAME; +use itertools::Itertools; use crate::executor::explain::PlanStatsInfo; +use crate::executor::physical_plan::PhysicalPlanDeriveHandle; use crate::executor::physical_plans::common::SortDesc; use crate::executor::physical_plans::WindowPartition; use crate::executor::physical_plans::WindowPartitionTopN; use crate::executor::physical_plans::WindowPartitionTopNFunc; -use crate::executor::{IPhysicalPlan, PhysicalPlan, PhysicalPlanMeta}; +use crate::executor::IPhysicalPlan; +use crate::executor::PhysicalPlan; use crate::executor::PhysicalPlanBuilder; +use crate::executor::PhysicalPlanMeta; use crate::optimizer::ir::SExpr; use crate::plans::WindowFuncType; use crate::ColumnSet; -use crate::executor::physical_plan::PhysicalPlanDeriveHandle; use crate::IndexType; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] @@ -99,11 +101,13 @@ impl IPhysicalPlan for Sort { Ok(DataSchemaRefExt::create(fields)) } - fn children<'a>(&'a self) -> Box> + 'a> { + fn children<'a>(&'a self) -> Box> + 'a> { Box::new(std::iter::once(&self.input)) } - fn children_mut<'a>(&'a mut self) -> Box> + 'a> { + fn children_mut<'a>( + &'a mut self, + ) -> Box> + 'a> { Box::new(std::iter::once(&mut self.input)) } @@ -124,11 +128,13 @@ impl IPhysicalPlan for Sort { if x.nulls_first { " NULLS FIRST" } else { "" }, ) }) - .join(", ") - ) + .join(", ")) } - fn derive_with(&self, handle: &mut Box) -> Box { + fn derive_with( + &self, + handle: &mut Box, + ) -> Box { let derive_input = self.input.derive_with(handle); match handle.derive(self, vec![derive_input]) { diff --git a/src/query/sql/src/executor/physical_plans/physical_table_scan.rs b/src/query/sql/src/executor/physical_plans/physical_table_scan.rs index eaef4fed1daa2..92480efdf2515 100644 --- a/src/query/sql/src/executor/physical_plans/physical_table_scan.rs +++ b/src/query/sql/src/executor/physical_plans/physical_table_scan.rs @@ -13,16 +13,20 @@ // limitations under the License. use std::collections::btree_map::Entry; -use std::collections::{BTreeMap, HashMap}; +use std::collections::BTreeMap; +use std::collections::HashMap; use std::collections::HashSet; use std::sync::Arc; use databend_common_ast::parser::token::TokenKind; use databend_common_ast::parser::tokenize_sql; use databend_common_catalog::catalog::CatalogManager; -use databend_common_catalog::plan::{DataSourceInfo, DataSourcePlan, PartStatistics, PartitionsShuffleKind}; +use databend_common_catalog::plan::DataSourceInfo; +use databend_common_catalog::plan::DataSourcePlan; use databend_common_catalog::plan::Filters; use databend_common_catalog::plan::InternalColumn; +use databend_common_catalog::plan::PartStatistics; +use databend_common_catalog::plan::PartitionsShuffleKind; use databend_common_catalog::plan::PrewhereInfo; use databend_common_catalog::plan::Projection; use databend_common_catalog::plan::PushDownInfo; @@ -51,12 +55,15 @@ use rand::distributions::Distribution; use rand::thread_rng; use crate::binder::INTERNAL_COLUMN_FACTORY; -use crate::executor::{cast_expr_to_non_null_boolean, IPhysicalPlan, PhysicalPlanMeta}; +use crate::executor::cast_expr_to_non_null_boolean; use crate::executor::explain::PlanStatsInfo; +use crate::executor::physical_plan::PhysicalPlanDeriveHandle; use crate::executor::physical_plans::AddStreamColumn; use crate::executor::table_read_plan::ToReadDataSourcePlan; +use crate::executor::IPhysicalPlan; use crate::executor::PhysicalPlan; use crate::executor::PhysicalPlanBuilder; +use crate::executor::PhysicalPlanMeta; use crate::plans::FunctionCall; use crate::BaseTableColumn; use crate::ColumnEntry; @@ -70,7 +77,6 @@ use crate::TypeCheck; use crate::VirtualColumn; use crate::DUMMY_COLUMN_INDEX; use crate::DUMMY_TABLE_INDEX; -use crate::executor::physical_plan::PhysicalPlanDeriveHandle; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct TableScan { @@ -97,7 +103,8 @@ impl IPhysicalPlan for TableScan { fn output_schema(&self) -> Result { let schema = self.source.schema(); let mut fields = Vec::with_capacity(self.name_mapping.len()); - let mut name_and_ids = self.name_mapping + let mut name_and_ids = self + .name_mapping .iter() .map(|(name, id)| { let index = schema.index_of(name)?; @@ -164,11 +171,14 @@ impl IPhysicalPlan for TableScan { .source .statistics .partitions_total - .to_string()]) + .to_string()]), ])) } - fn derive_with(&self, handle: &mut Box) -> Box { + fn derive_with( + &self, + handle: &mut Box, + ) -> Box { match handle.derive(self, vec![]) { Ok(v) => v, Err(children) => { @@ -180,7 +190,14 @@ impl IPhysicalPlan for TableScan { } impl TableScan { - pub fn new(scan_id: usize, name_mapping: BTreeMap, source: Box, table_index: Option, stat_info: Option, internal_column: Option>) -> Box { + pub fn new( + scan_id: usize, + name_mapping: BTreeMap, + source: Box, + table_index: Option, + stat_info: Option, + internal_column: Option>, + ) -> Box { let name = match &source.source_info { DataSourceInfo::TableSource(_) => "TableScan".to_string(), DataSourceInfo::StageSource(_) => "StageScan".to_string(), @@ -200,7 +217,6 @@ impl TableScan { }) } - pub fn output_fields( schema: TableSchemaRef, name_mapping: &BTreeMap, @@ -298,8 +314,8 @@ impl PhysicalPlanBuilder { } } ColumnEntry::InternalColumn(TableInternalColumn { - internal_column, .. - }) => { + internal_column, .. + }) => { project_internal_columns.insert(*index, internal_column.to_owned()); } ColumnEntry::VirtualColumn(virtual_column) => { @@ -620,14 +636,14 @@ impl PhysicalPlanBuilder { let column = metadata.column(item.index); let (name, data_type) = match column { ColumnEntry::BaseTableColumn(BaseTableColumn { - column_name, - data_type, - .. - }) => (column_name.clone(), DataType::from(data_type)), + column_name, + data_type, + .. + }) => (column_name.clone(), DataType::from(data_type)), ColumnEntry::InternalColumn(TableInternalColumn { - internal_column, - .. - }) => ( + internal_column, + .. + }) => ( internal_column.column_name().to_owned(), internal_column.data_type(), ), @@ -799,7 +815,7 @@ impl PhysicalPlanBuilder { cast_expr_to_non_null_boolean(pred.as_expr()?.project_column_ref(|col| { output_schema.index_of(&col.index.to_string()).unwrap() }))? - .as_remote_expr(), + .as_remote_expr(), ) }) .transpose()?; @@ -837,7 +853,7 @@ impl PhysicalPlanBuilder { pub fn build_projection<'a>( metadata: &Metadata, schema: &TableSchema, - columns: impl Iterator, + columns: impl Iterator, has_inner_column: bool, ignore_internal_column: bool, add_virtual_source_column: bool, @@ -856,16 +872,16 @@ impl PhysicalPlanBuilder { } ColumnEntry::DerivedColumn(DerivedColumn { alias, .. }) => alias, ColumnEntry::InternalColumn(TableInternalColumn { - internal_column, .. - }) => { + internal_column, .. + }) => { if ignore_internal_column { continue; } internal_column.column_name() } ColumnEntry::VirtualColumn(VirtualColumn { - source_column_name, .. - }) => { + source_column_name, .. + }) => { if add_virtual_source_column { virtual_col_indices .insert(schema.index_of(source_column_name).unwrap()); @@ -893,10 +909,10 @@ impl PhysicalPlanBuilder { let column = metadata.column(*index); match column { ColumnEntry::BaseTableColumn(BaseTableColumn { - column_name, - path_indices, - .. - }) => match path_indices { + column_name, + path_indices, + .. + }) => match path_indices { Some(path_indices) => { col_indices.insert(column.index(), path_indices.to_vec()); } @@ -915,8 +931,8 @@ impl PhysicalPlanBuilder { } } ColumnEntry::VirtualColumn(VirtualColumn { - source_column_name, .. - }) => { + source_column_name, .. + }) => { if add_virtual_source_column { let idx = schema.index_of(source_column_name).unwrap(); col_indices.insert(idx, vec![idx]); diff --git a/src/query/sql/src/executor/physical_plans/physical_udf.rs b/src/query/sql/src/executor/physical_plans/physical_udf.rs index 5c2f47cc061ff..0abddc607dbf2 100644 --- a/src/query/sql/src/executor/physical_plans/physical_udf.rs +++ b/src/query/sql/src/executor/physical_plans/physical_udf.rs @@ -13,7 +13,7 @@ // limitations under the License. use std::collections::BTreeMap; -use itertools::Itertools; + use databend_common_catalog::plan::DataSourcePlan; use databend_common_exception::ErrorCode; use databend_common_exception::Result; @@ -22,14 +22,17 @@ use databend_common_expression::DataField; use databend_common_expression::DataSchemaRef; use databend_common_expression::DataSchemaRefExt; use databend_common_functions::BUILTIN_FUNCTIONS; +use itertools::Itertools; use crate::executor::explain::PlanStatsInfo; -use crate::executor::{IPhysicalPlan, PhysicalPlan, PhysicalPlanMeta}; +use crate::executor::physical_plan::PhysicalPlanDeriveHandle; +use crate::executor::IPhysicalPlan; +use crate::executor::PhysicalPlan; use crate::executor::PhysicalPlanBuilder; +use crate::executor::PhysicalPlanMeta; use crate::optimizer::ir::SExpr; use crate::plans::UDFType; use crate::ColumnSet; -use crate::executor::physical_plan::PhysicalPlanDeriveHandle; use crate::IndexType; use crate::ScalarExpr; @@ -64,11 +67,13 @@ impl IPhysicalPlan for Udf { Ok(DataSchemaRefExt::create(fields)) } - fn children<'a>(&'a self) -> Box> + 'a> { + fn children<'a>(&'a self) -> Box> + 'a> { Box::new(std::iter::once(&self.input)) } - fn children_mut<'a>(&'a mut self) -> Box> + 'a> { + fn children_mut<'a>( + &'a mut self, + ) -> Box> + 'a> { Box::new(std::iter::once(&mut self.input)) } @@ -85,7 +90,10 @@ impl IPhysicalPlan for Udf { .join(", ")) } - fn derive_with(&self, handle: &mut Box) -> Box { + fn derive_with( + &self, + handle: &mut Box, + ) -> Box { let derive_input = self.input.derive_with(handle); match handle.derive(self, vec![derive_input]) { diff --git a/src/query/sql/src/executor/physical_plans/physical_union_all.rs b/src/query/sql/src/executor/physical_plans/physical_union_all.rs index 31eebbf8e91d6..5dff25d4cce82 100644 --- a/src/query/sql/src/executor/physical_plans/physical_union_all.rs +++ b/src/query/sql/src/executor/physical_plans/physical_union_all.rs @@ -12,20 +12,22 @@ // See the License for the specific language governing permissions and // limitations under the License. -use itertools::Itertools; use databend_common_exception::Result; use databend_common_expression::DataField; use databend_common_expression::DataSchema; use databend_common_expression::DataSchemaRef; use databend_common_expression::DataSchemaRefExt; use databend_common_expression::RemoteExpr; +use itertools::Itertools; use crate::executor::explain::PlanStatsInfo; -use crate::executor::{IPhysicalPlan, PhysicalPlan, PhysicalPlanMeta}; +use crate::executor::physical_plan::PhysicalPlanDeriveHandle; +use crate::executor::IPhysicalPlan; +use crate::executor::PhysicalPlan; use crate::executor::PhysicalPlanBuilder; +use crate::executor::PhysicalPlanMeta; use crate::optimizer::ir::SExpr; use crate::ColumnSet; -use crate::executor::physical_plan::PhysicalPlanDeriveHandle; use crate::IndexType; use crate::ScalarExpr; use crate::TypeCheck; @@ -58,11 +60,13 @@ impl IPhysicalPlan for UnionAll { Ok(self.schema.clone()) } - fn children<'a>(&'a self) -> Box> + 'a> { + fn children<'a>(&'a self) -> Box> + 'a> { Box::new(std::iter::once(&self.left).chain(std::iter::once(&self.right))) } - fn children_mut<'a>(&'a mut self) -> Box> + 'a> { + fn children_mut<'a>( + &'a mut self, + ) -> Box> + 'a> { Box::new(std::iter::once(&mut self.left).chain(std::iter::once(&mut self.right))) } @@ -75,7 +79,10 @@ impl IPhysicalPlan for UnionAll { .join(", ")) } - fn derive_with(&self, handle: &mut Box) -> Box { + fn derive_with( + &self, + handle: &mut Box, + ) -> Box { let derive_left = self.left.derive_with(handle); let derive_right = self.right.derive_with(handle); diff --git a/src/query/sql/src/executor/physical_plans/physical_window.rs b/src/query/sql/src/executor/physical_plans/physical_window.rs index 92136522cd19d..1e4dfd95c75fa 100644 --- a/src/query/sql/src/executor/physical_plans/physical_window.rs +++ b/src/query/sql/src/executor/physical_plans/physical_window.rs @@ -13,6 +13,7 @@ // limitations under the License. use std::fmt::Display; + use databend_common_catalog::plan::DataSourcePlan; use databend_common_exception::ErrorCode; use databend_common_exception::Result; @@ -32,17 +33,19 @@ use databend_common_functions::BUILTIN_FUNCTIONS; use crate::binder::wrap_cast; use crate::executor::explain::PlanStatsInfo; +use crate::executor::physical_plan::PhysicalPlanDeriveHandle; use crate::executor::physical_plans::common::AggregateFunctionDesc; use crate::executor::physical_plans::common::AggregateFunctionSignature; use crate::executor::physical_plans::common::SortDesc; -use crate::executor::{IPhysicalPlan, PhysicalPlan, PhysicalPlanMeta}; +use crate::executor::IPhysicalPlan; +use crate::executor::PhysicalPlan; use crate::executor::PhysicalPlanBuilder; +use crate::executor::PhysicalPlanMeta; use crate::optimizer::ir::SExpr; use crate::plans::WindowFuncFrame; use crate::plans::WindowFuncFrameBound; use crate::plans::WindowFuncType; use crate::ColumnSet; -use crate::executor::physical_plan::PhysicalPlanDeriveHandle; use crate::IndexType; use crate::ScalarExpr; use crate::TypeCheck; @@ -80,11 +83,13 @@ impl IPhysicalPlan for Window { Ok(DataSchemaRefExt::create(fields)) } - fn children<'a>(&'a self) -> Box> + 'a> { + fn children<'a>(&'a self) -> Box> + 'a> { Box::new(std::iter::once(&self.input)) } - fn children_mut<'a>(&'a mut self) -> Box> + 'a> { + fn children_mut<'a>( + &'a mut self, + ) -> Box> + 'a> { Box::new(std::iter::once(&mut self.input)) } @@ -115,10 +120,16 @@ impl IPhysicalPlan for Window { .collect::>() .join(", "); - Ok(format!("partition by {}, order by {}", partition_by, order_by)) + Ok(format!( + "partition by {}, order by {}", + partition_by, order_by + )) } - fn derive_with(&self, handle: &mut Box) -> Box { + fn derive_with( + &self, + handle: &mut Box, + ) -> Box { let derive_input = self.input.derive_with(handle); match handle.derive(self, vec![derive_input]) { diff --git a/src/query/sql/src/executor/physical_plans/physical_window_partition.rs b/src/query/sql/src/executor/physical_plans/physical_window_partition.rs index 8eba8f7e69fa0..e15837a8b723a 100644 --- a/src/query/sql/src/executor/physical_plans/physical_window_partition.rs +++ b/src/query/sql/src/executor/physical_plans/physical_window_partition.rs @@ -17,9 +17,11 @@ use databend_common_exception::Result; use databend_common_expression::DataSchemaRef; use crate::executor::explain::PlanStatsInfo; -use crate::executor::physical_plans::SortDesc; -use crate::executor::{IPhysicalPlan, PhysicalPlan, PhysicalPlanMeta}; use crate::executor::physical_plan::PhysicalPlanDeriveHandle; +use crate::executor::physical_plans::SortDesc; +use crate::executor::IPhysicalPlan; +use crate::executor::PhysicalPlan; +use crate::executor::PhysicalPlanMeta; use crate::IndexType; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] @@ -44,11 +46,13 @@ impl IPhysicalPlan for WindowPartition { &mut self.meta } - fn children<'a>(&'a self) -> Box> + 'a> { + fn children<'a>(&'a self) -> Box> + 'a> { Box::new(std::iter::once(&self.input)) } - fn children_mut<'a>(&'a mut self) -> Box> + 'a> { + fn children_mut<'a>( + &'a mut self, + ) -> Box> + 'a> { Box::new(std::iter::once(&mut self.input)) } @@ -57,7 +61,10 @@ impl IPhysicalPlan for WindowPartition { self.input.try_find_single_data_source() } - fn derive_with(&self, handle: &mut Box) -> Box { + fn derive_with( + &self, + handle: &mut Box, + ) -> Box { let derive_input = self.input.derive_with(handle); match handle.derive(self, vec![derive_input]) { From adbf021e866ce4a6df16998fcf0c6a884d05d4b9 Mon Sep 17 00:00:00 2001 From: zhang2014 Date: Thu, 3 Jul 2025 12:10:40 +0800 Subject: [PATCH 6/8] refactor(query): use trait to refactor physical plan --- .../src/interpreters/interpreter_explain.rs | 32 +- .../interpreters/interpreter_index_refresh.rs | 4 +- .../interpreter_insert_multi_table.rs | 80 +- .../src/interpreters/interpreter_mutation.rs | 3 +- .../src/interpreters/interpreter_select.rs | 3 +- .../src/schedulers/fragments/fragmenter.rs | 389 ++-- .../src/schedulers/fragments/plan_fragment.rs | 30 +- .../fragments/query_fragment_actions.rs | 9 +- src/query/service/src/schedulers/scheduler.rs | 9 +- .../flight/v1/packets/packet_fragment.rs | 6 +- src/query/sql/src/executor/format.rs | 1711 +---------------- src/query/sql/src/executor/mod.rs | 4 +- src/query/sql/src/executor/physical_plan.rs | 88 +- .../physical_add_stream_column.rs | 33 +- .../physical_aggregate_expand.rs | 65 +- .../physical_aggregate_final.rs | 109 +- .../physical_aggregate_partial.rs | 67 +- .../physical_plans/physical_async_func.rs | 50 +- .../physical_plans/physical_broadcast.rs | 45 +- .../physical_plans/physical_cache_scan.rs | 42 +- .../physical_column_mutation.rs | 30 +- .../physical_plans/physical_commit_sink.rs | 22 +- .../physical_plans/physical_compact_source.rs | 18 +- .../physical_constant_table_scan.rs | 41 +- .../physical_copy_into_location.rs | 35 +- .../physical_copy_into_table.rs | 54 +- .../physical_distributed_insert_select.rs | 22 +- .../physical_plans/physical_eval_scalar.rs | 63 +- .../physical_plans/physical_exchange.rs | 63 +- .../physical_plans/physical_exchange_sink.rs | 47 +- .../physical_exchange_source.rs | 41 +- .../physical_expression_scan.rs | 51 +- .../physical_plans/physical_filter.rs | 59 +- .../physical_plans/physical_hash_join.rs | 143 +- .../executor/physical_plans/physical_limit.rs | 63 +- .../physical_multi_table_insert.rs | 293 +-- .../physical_plans/physical_mutation.rs | 53 +- .../physical_mutation_into_organize.rs | 30 +- .../physical_mutation_into_split.rs | 30 +- .../physical_mutation_manipulate.rs | 115 +- .../physical_mutation_source.rs | 58 +- .../physical_plans/physical_project_set.rs | 59 +- .../physical_plans/physical_r_cte_scan.rs | 16 +- .../physical_plans/physical_range_join.rs | 87 +- .../physical_plans/physical_recluster.rs | 36 +- .../physical_replace_async_source.rs | 16 +- .../physical_replace_deduplicate.rs | 22 +- .../physical_plans/physical_replace_into.rs | 22 +- .../physical_plans/physical_row_fetch.rs | 58 +- .../executor/physical_plans/physical_sort.rs | 74 +- .../physical_plans/physical_table_scan.rs | 152 +- .../executor/physical_plans/physical_udf.rs | 63 +- .../physical_plans/physical_union_all.rs | 54 +- .../physical_plans/physical_window.rs | 76 +- .../physical_window_partition.rs | 61 +- .../src/planner/execution/query_executor.rs | 7 +- 56 files changed, 2107 insertions(+), 2806 deletions(-) diff --git a/src/query/service/src/interpreters/interpreter_explain.rs b/src/query/service/src/interpreters/interpreter_explain.rs index 88da04bed02bf..708f0388ed71c 100644 --- a/src/query/service/src/interpreters/interpreter_explain.rs +++ b/src/query/service/src/interpreters/interpreter_explain.rs @@ -35,6 +35,7 @@ use databend_common_pipeline_core::ExecutionInfo; use databend_common_pipeline_core::Pipeline; use databend_common_sql::binder::ExplainConfig; use databend_common_sql::executor::format_partial_tree; +use databend_common_sql::executor::IPhysicalPlan; use databend_common_sql::executor::MutationBuildInfo; use databend_common_sql::plans::Mutation; use databend_common_sql::BindContext; @@ -164,22 +165,11 @@ impl Interpreter for ExplainInterpreter { _ => self.explain_plan(&self.plan)?, }, - ExplainKind::Join => match &self.plan { - Plan::Query { - s_expr, - metadata, - bind_context, - .. - } => { - let ctx = self.ctx.clone(); - let mut builder = PhysicalPlanBuilder::new(metadata.clone(), ctx, true); - let plan = builder.build(s_expr, bind_context.column_set()).await?; - self.explain_join_order(&plan, metadata)? - } - _ => Err(ErrorCode::Unimplemented( + ExplainKind::Join => { + return Err(ErrorCode::Unimplemented( "Unsupported EXPLAIN JOIN statement", - ))?, - }, + )); + } ExplainKind::AnalyzePlan | ExplainKind::Graphical => match &self.plan { Plan::Query { @@ -324,7 +314,7 @@ impl ExplainInterpreter { pub async fn explain_physical_plan( &self, - plan: &PhysicalPlan, + plan: &Box, metadata: &MetadataRef, formatted_ast: &Option, ) -> Result> { @@ -360,8 +350,9 @@ impl ExplainInterpreter { } } + let metadata = metadata.read(); let result = plan - .format(metadata.clone(), Default::default())? + .format(&metadata, Default::default())? .format_pretty()?; let line_split_result: Vec<&str> = result.lines().collect(); let formatted_plan = StringType::from_data(line_split_result); @@ -479,10 +470,9 @@ impl ExplainInterpreter { if !pruned_partitions_stats.is_empty() { plan.set_pruning_stats(&mut pruned_partitions_stats); } - let result = if self.partial { - format_partial_tree(&plan, metadata, &query_profiles)?.format_pretty()? - } else { - plan.format(metadata.clone(), query_profiles.clone())? + let result = { + let metadata = metadata.read(); + plan.format(&metadata, query_profiles.clone())? .format_pretty()? }; let line_split_result: Vec<&str> = result.lines().collect(); diff --git a/src/query/service/src/interpreters/interpreter_index_refresh.rs b/src/query/service/src/interpreters/interpreter_index_refresh.rs index 6f4ba43fce1a1..341150b8066e8 100644 --- a/src/query/service/src/interpreters/interpreter_index_refresh.rs +++ b/src/query/service/src/interpreters/interpreter_index_refresh.rs @@ -35,10 +35,10 @@ use databend_common_pipeline_transforms::processors::TransformPipelineHelper; use databend_common_sql::evaluator::BlockOperator; use databend_common_sql::evaluator::CompoundBlockOperator; use databend_common_sql::executor::physical_plans::TableScan; +use databend_common_sql::executor::DeriveHandle; use databend_common_sql::executor::IPhysicalPlan; use databend_common_sql::executor::PhysicalPlan; use databend_common_sql::executor::PhysicalPlanBuilder; -use databend_common_sql::executor::PhysicalPlanDeriveHandle; use databend_common_sql::plans::Plan; use databend_common_sql::plans::RefreshIndexPlan; use databend_common_sql::plans::RelOperator; @@ -382,7 +382,7 @@ struct ReadSourceDeriveHandle { source: DataSourcePlan, } -impl PhysicalPlanDeriveHandle for ReadSourceDeriveHandle { +impl DeriveHandle for ReadSourceDeriveHandle { fn derive( &mut self, v: &Box, diff --git a/src/query/service/src/interpreters/interpreter_insert_multi_table.rs b/src/query/service/src/interpreters/interpreter_insert_multi_table.rs index c3bd900836bc1..2f085cf4548b8 100644 --- a/src/query/service/src/interpreters/interpreter_insert_multi_table.rs +++ b/src/query/service/src/interpreters/interpreter_insert_multi_table.rs @@ -39,8 +39,10 @@ use databend_common_sql::executor::physical_plans::MultiInsertEvalScalar; use databend_common_sql::executor::physical_plans::MutationKind; use databend_common_sql::executor::physical_plans::SerializableTable; use databend_common_sql::executor::physical_plans::ShuffleStrategy; +use databend_common_sql::executor::IPhysicalPlan; use databend_common_sql::executor::PhysicalPlan; use databend_common_sql::executor::PhysicalPlanBuilder; +use databend_common_sql::executor::PhysicalPlanMeta; use databend_common_sql::plans::Else; use databend_common_sql::plans::FunctionCall; use databend_common_sql::plans::InsertMultiTable; @@ -130,7 +132,7 @@ impl Interpreter for InsertMultiTableInterpreter { } impl InsertMultiTableInterpreter { - pub async fn build_physical_plan(&self) -> Result { + pub async fn build_physical_plan(&self) -> Result> { let (mut root, _) = self.build_source_physical_plan().await?; let update_stream_meta = dml_build_update_stream_req(self.ctx.clone()).await?; let source_schema = root.output_schema()?; @@ -174,69 +176,71 @@ impl InsertMultiTableInterpreter { let fill_and_reorders = branches.build_fill_and_reorder(self.ctx.clone()).await?; let group_ids = branches.build_group_ids(); - root = PhysicalPlan::Duplicate(Box::new(Duplicate { - plan_id: 0, - input: Box::new(root), + root = Box::new(Duplicate { + input: root, n: branches.len(), - })); + meta: PhysicalPlanMeta::new("Duplicate"), + }); let shuffle_strategy = ShuffleStrategy::Transpose(branches.len()); - root = PhysicalPlan::Shuffle(Box::new(Shuffle { - plan_id: 0, - input: Box::new(root), + root = Box::new(Shuffle { + input: root, strategy: shuffle_strategy, - })); + meta: PhysicalPlanMeta::new("Shuffle"), + }); - root = PhysicalPlan::ChunkFilter(Box::new(ChunkFilter { - plan_id: 0, - input: Box::new(root), + root = Box::new(ChunkFilter { predicates, - })); + input: root, + meta: PhysicalPlanMeta::new("ChunkFilter"), + }); - root = PhysicalPlan::ChunkEvalScalar(Box::new(ChunkEvalScalar { - plan_id: 0, - input: Box::new(root), + root = Box::new(ChunkEvalScalar { eval_scalars, - })); + input: root, + meta: PhysicalPlanMeta::new("ChunkEvalScalar"), + }); - root = PhysicalPlan::ChunkCastSchema(Box::new(ChunkCastSchema { - plan_id: 0, - input: Box::new(root), + root = Box::new(ChunkCastSchema { cast_schemas, - })); + input: root, + meta: PhysicalPlanMeta::new("ChunkCastSchema"), + }); - root = PhysicalPlan::ChunkFillAndReorder(Box::new(ChunkFillAndReorder { - plan_id: 0, - input: Box::new(root), + root = Box::new(ChunkFillAndReorder { fill_and_reorders, - })); + input: root, + meta: PhysicalPlanMeta::new("ChunkFillAndReorder"), + }); - root = PhysicalPlan::ChunkAppendData(Box::new(ChunkAppendData { - plan_id: 0, - input: Box::new(root), + root = Box::new(ChunkAppendData { + input: root, target_tables: serializable_tables.clone(), - })); + meta: PhysicalPlanMeta::new("ChunkAppendData"), + }); - root = PhysicalPlan::ChunkMerge(Box::new(ChunkMerge { - plan_id: 0, - input: Box::new(root), + root = Box::new(ChunkMerge { group_ids, - })); + input: root, + meta: PhysicalPlanMeta::new("ChunkMerge"), + }); - root = PhysicalPlan::ChunkCommitInsert(Box::new(ChunkCommitInsert { - plan_id: 0, - input: Box::new(root), + root = Box::new(ChunkCommitInsert { update_stream_meta, + + input: root, overwrite: self.plan.overwrite, deduplicated_label: None, targets: deduplicated_serializable_tables, - })); + meta: PhysicalPlanMeta::new("ChunkCommitInsert"), + }); + let mut next_plan_id = 0; root.adjust_plan_id(&mut next_plan_id); Ok(root) } - async fn build_source_physical_plan(&self) -> Result<(PhysicalPlan, MetadataRef)> { + async fn build_source_physical_plan(&self) -> Result<(Box, MetadataRef)> { match &self.plan.input_source { Plan::Query { s_expr, diff --git a/src/query/service/src/interpreters/interpreter_mutation.rs b/src/query/service/src/interpreters/interpreter_mutation.rs index 2e41ed14faaec..023929151ef1f 100644 --- a/src/query/service/src/interpreters/interpreter_mutation.rs +++ b/src/query/service/src/interpreters/interpreter_mutation.rs @@ -32,6 +32,7 @@ use databend_common_sql::binder::MutationStrategy; use databend_common_sql::binder::MutationType; use databend_common_sql::executor::physical_plans::create_push_down_filters; use databend_common_sql::executor::physical_plans::MutationKind; +use databend_common_sql::executor::IPhysicalPlan; use databend_common_sql::executor::MutationBuildInfo; use databend_common_sql::executor::PhysicalPlan; use databend_common_sql::executor::PhysicalPlanBuilder; @@ -167,7 +168,7 @@ impl MutationInterpreter { &self, mutation: &Mutation, dry_run: bool, - ) -> Result { + ) -> Result> { // Prepare MutationBuildInfo for PhysicalPlanBuilder to build DataMutation physical plan. let mutation_build_info = build_mutation_info(self.ctx.clone(), mutation, dry_run).await?; // Build physical plan. diff --git a/src/query/service/src/interpreters/interpreter_select.rs b/src/query/service/src/interpreters/interpreter_select.rs index 4d2941f6aea66..73e43fc90f51f 100644 --- a/src/query/service/src/interpreters/interpreter_select.rs +++ b/src/query/service/src/interpreters/interpreter_select.rs @@ -33,6 +33,7 @@ use databend_common_pipeline_core::PipeItem; use databend_common_pipeline_core::Pipeline; use databend_common_pipeline_transforms::processors::TransformDummy; use databend_common_sql::executor::physical_plans::FragmentKind; +use databend_common_sql::executor::IPhysicalPlan; use databend_common_sql::executor::PhysicalPlan; use databend_common_sql::parse_result_scan_args; use databend_common_sql::ColumnBinding; @@ -110,7 +111,7 @@ impl SelectInterpreter { #[inline] #[async_backtrace::framed] - pub async fn build_physical_plan(&self) -> Result { + pub async fn build_physical_plan(&self) -> Result> { let mut builder = PhysicalPlanBuilder::new(self.metadata.clone(), self.ctx.clone(), false); self.ctx .set_status_info("[SELECT-INTERP] Building physical plan"); diff --git a/src/query/service/src/schedulers/fragments/fragmenter.rs b/src/query/service/src/schedulers/fragments/fragmenter.rs index 7f7cd877e87e8..3446e35a00193 100644 --- a/src/query/service/src/schedulers/fragments/fragmenter.rs +++ b/src/query/service/src/schedulers/fragments/fragmenter.rs @@ -31,6 +31,8 @@ use databend_common_sql::executor::physical_plans::Recluster; use databend_common_sql::executor::physical_plans::ReplaceInto; use databend_common_sql::executor::physical_plans::TableScan; use databend_common_sql::executor::physical_plans::UnionAll; +use databend_common_sql::executor::DeriveHandle; +use databend_common_sql::executor::IPhysicalPlan; use crate::clusters::ClusterHelper; use crate::schedulers::fragments::plan_fragment::FragmentType; @@ -46,9 +48,9 @@ use crate::sql::executor::PhysicalPlan; /// Visitor to split a `PhysicalPlan` into fragments. pub struct Fragmenter { ctx: Arc, - fragments: Vec, - query_id: String, state: State, + query_id: String, + fragments: Vec, } /// A state to track if is visiting a source fragment, useful when building fragments. @@ -120,8 +122,8 @@ impl Fragmenter { } } - pub fn build_fragment(mut self, plan: &PhysicalPlan) -> Result { - let root = self.replace(plan)?; + pub fn build_fragment(mut self, plan: &Box) -> Result { + let root = plan.derive_with(&mut self); let fragment_type = match plan { PhysicalPlan::BroadcastSink(_) => FragmentType::Intermediate, _ => FragmentType::Root, @@ -153,191 +155,200 @@ impl Fragmenter { } } -impl PhysicalPlanReplacer for Fragmenter { - fn replace_recluster(&mut self, plan: &Recluster) -> Result { - self.state = State::Recluster; - Ok(PhysicalPlan::Recluster(Box::new(plan.clone()))) - } - - fn replace_table_scan(&mut self, plan: &TableScan) -> Result { - self.state = State::SelectLeaf; - Ok(PhysicalPlan::TableScan(plan.clone())) - } - - fn replace_constant_table_scan(&mut self, plan: &ConstantTableScan) -> Result { - self.state = State::SelectLeaf; - Ok(PhysicalPlan::ConstantTableScan(plan.clone())) - } - - fn replace_hash_join(&mut self, plan: &HashJoin) -> Result { - let mut fragments = vec![]; - let build_input = self.replace(plan.build.as_ref())?; - - // Consume current fragments to prevent them being consumed by `probe_input`. - fragments.append(&mut self.fragments); - let probe_input = self.replace(plan.probe.as_ref())?; - fragments.append(&mut self.fragments); - - self.fragments = fragments; - - Ok(PhysicalPlan::HashJoin(HashJoin { - plan_id: plan.plan_id, - projections: plan.projections.clone(), - probe_projections: plan.probe_projections.clone(), - build_projections: plan.build_projections.clone(), - build: Box::new(build_input), - probe: Box::new(probe_input), - build_keys: plan.build_keys.clone(), - probe_keys: plan.probe_keys.clone(), - is_null_equal: plan.is_null_equal.clone(), - non_equi_conditions: plan.non_equi_conditions.clone(), - join_type: plan.join_type.clone(), - marker_index: plan.marker_index, - from_correlated_subquery: plan.from_correlated_subquery, - probe_to_build: plan.probe_to_build.clone(), - output_schema: plan.output_schema.clone(), - need_hold_hash_table: plan.need_hold_hash_table, - stat_info: plan.stat_info.clone(), - single_to_inner: plan.single_to_inner.clone(), - build_side_cache_info: plan.build_side_cache_info.clone(), - runtime_filter: plan.runtime_filter.clone(), - broadcast_id: plan.broadcast_id, - })) - } - - fn replace_exchange(&mut self, plan: &Exchange) -> Result { - // Recursively rewrite input - let input = self.replace(plan.input.as_ref())?; - let input_schema = input.output_schema()?; - - let plan_id = plan.plan_id; - - let source_fragment_id = self.ctx.get_fragment_id(); - let plan = PhysicalPlan::ExchangeSink(ExchangeSink { - // TODO(leiysky): we reuse the plan id here, - // should generate a new one for the sink. - plan_id, - - input: Box::new(input), - schema: input_schema.clone(), - kind: plan.kind.clone(), - keys: plan.keys.clone(), - - query_id: self.query_id.clone(), - - // We will connect the fragments later, so we just - // set the fragment id to a invalid value here. - destination_fragment_id: usize::MAX, - ignore_exchange: plan.ignore_exchange, - allow_adjust_parallelism: plan.allow_adjust_parallelism, - }); - let fragment_type = match self.state { - State::SelectLeaf => FragmentType::Source, - State::MutationSource => FragmentType::MutationSource, - State::Other => FragmentType::Intermediate, - State::ReplaceInto => FragmentType::ReplaceInto, - State::Compact => FragmentType::Compact, - State::Recluster => FragmentType::Recluster, +impl DeriveHandle for Fragmenter { + fn derive( + &mut self, + v: &Box, + children: Vec>, + ) -> std::result::Result, Vec>> { + let Some(compact_source) = v.down_cast::() else { + return Err(children); }; - self.state = State::Other; - let exchange = Self::get_exchange(self.ctx.clone(), &plan)?; - - let mut source_fragment = PlanFragment { - plan, - fragment_type, - - fragment_id: source_fragment_id, - exchange, - query_id: self.query_id.clone(), - - source_fragments: self.fragments.drain(..).collect(), - }; - - // Fill the destination_fragment_id for source fragments of `source_fragment`. - Self::resolve_fragment_connection(&mut source_fragment); - - self.fragments.push(source_fragment); - - Ok(PhysicalPlan::ExchangeSource(ExchangeSource { - // TODO(leiysky): we reuse the plan id here, - // should generate a new one for the source. - plan_id, - - schema: input_schema, - query_id: self.query_id.clone(), - - source_fragment_id, - })) - } - - fn replace_union(&mut self, plan: &UnionAll) -> Result { - let mut fragments = vec![]; - let left_input = self.replace(plan.left.as_ref())?; - let left_state = self.state.clone(); - - // Consume current fragments to prevent them being consumed by `right_input`. - fragments.append(&mut self.fragments); - let right_input = self.replace(plan.right.as_ref())?; - let right_state = self.state.clone(); - - fragments.append(&mut self.fragments); - self.fragments = fragments; - - // If any of the input is a source fragment, the union all is a source fragment. - if left_state == State::SelectLeaf || right_state == State::SelectLeaf { - self.state = State::SelectLeaf; - } else { - self.state = State::Other; - } - - Ok(PhysicalPlan::UnionAll(UnionAll { - left: Box::new(left_input), - right: Box::new(right_input), - ..plan.clone() - })) - } - - // TODO(Sky): remove redundant code - fn replace_copy_into_table(&mut self, plan: &CopyIntoTable) -> Result { - match &plan.source { - CopyIntoTableSource::Stage(_) => { - self.state = State::SelectLeaf; - Ok(PhysicalPlan::CopyIntoTable(Box::new(plan.clone()))) - } - CopyIntoTableSource::Query(query_physical_plan) => { - let input = self.replace(query_physical_plan)?; - Ok(PhysicalPlan::CopyIntoTable(Box::new(CopyIntoTable { - source: CopyIntoTableSource::Query(Box::new(input)), - ..plan.clone() - }))) - } - } - } - - fn replace_compact_source(&mut self, plan: &CompactSource) -> Result { - self.state = State::Compact; - Ok(PhysicalPlan::CompactSource(Box::new(plan.clone()))) - } - - fn replace_replace_into(&mut self, plan: &ReplaceInto) -> Result { - let input = self.replace(&plan.input)?; - self.state = State::ReplaceInto; - Ok(PhysicalPlan::ReplaceInto(Box::new(ReplaceInto { - input: Box::new(input), - ..plan.clone() - }))) - } - - fn replace_mutation_source(&mut self, plan: &MutationSource) -> Result { - self.state = State::MutationSource; - Ok(PhysicalPlan::MutationSource(plan.clone())) - } - - fn replace_mutation(&mut self, plan: &Mutation) -> Result { - let input = self.replace(&plan.input)?; - Ok(PhysicalPlan::Mutation(Box::new(Mutation { - input: Box::new(input), - ..plan.clone() - }))) } } +// impl PhysicalPlanReplacer for Fragmenter { +// fn replace_recluster(&mut self, plan: &Recluster) -> Result { +// self.state = State::Recluster; +// Ok(PhysicalPlan::Recluster(Box::new(plan.clone()))) +// } +// +// fn replace_table_scan(&mut self, plan: &TableScan) -> Result { +// self.state = State::SelectLeaf; +// Ok(PhysicalPlan::TableScan(plan.clone())) +// } +// +// fn replace_constant_table_scan(&mut self, plan: &ConstantTableScan) -> Result { +// self.state = State::SelectLeaf; +// Ok(PhysicalPlan::ConstantTableScan(plan.clone())) +// } +// +// fn replace_hash_join(&mut self, plan: &HashJoin) -> Result { +// let mut fragments = vec![]; +// let build_input = self.replace(plan.build.as_ref())?; +// +// // Consume current fragments to prevent them being consumed by `probe_input`. +// fragments.append(&mut self.fragments); +// let probe_input = self.replace(plan.probe.as_ref())?; +// fragments.append(&mut self.fragments); +// +// self.fragments = fragments; +// +// Ok(PhysicalPlan::HashJoin(HashJoin { +// plan_id: plan.plan_id, +// projections: plan.projections.clone(), +// probe_projections: plan.probe_projections.clone(), +// build_projections: plan.build_projections.clone(), +// build: Box::new(build_input), +// probe: Box::new(probe_input), +// build_keys: plan.build_keys.clone(), +// probe_keys: plan.probe_keys.clone(), +// is_null_equal: plan.is_null_equal.clone(), +// non_equi_conditions: plan.non_equi_conditions.clone(), +// join_type: plan.join_type.clone(), +// marker_index: plan.marker_index, +// from_correlated_subquery: plan.from_correlated_subquery, +// probe_to_build: plan.probe_to_build.clone(), +// output_schema: plan.output_schema.clone(), +// need_hold_hash_table: plan.need_hold_hash_table, +// stat_info: plan.stat_info.clone(), +// single_to_inner: plan.single_to_inner.clone(), +// build_side_cache_info: plan.build_side_cache_info.clone(), +// runtime_filter: plan.runtime_filter.clone(), +// broadcast_id: plan.broadcast_id, +// })) +// } +// +// fn replace_exchange(&mut self, plan: &Exchange) -> Result { +// // Recursively rewrite input +// let input = self.replace(plan.input.as_ref())?; +// let input_schema = input.output_schema()?; +// +// let plan_id = plan.plan_id; +// +// let source_fragment_id = self.ctx.get_fragment_id(); +// let plan = PhysicalPlan::ExchangeSink(ExchangeSink { +// // TODO(leiysky): we reuse the plan id here, +// // should generate a new one for the sink. +// plan_id, +// +// input: Box::new(input), +// schema: input_schema.clone(), +// kind: plan.kind.clone(), +// keys: plan.keys.clone(), +// +// query_id: self.query_id.clone(), +// +// // We will connect the fragments later, so we just +// // set the fragment id to a invalid value here. +// destination_fragment_id: usize::MAX, +// ignore_exchange: plan.ignore_exchange, +// allow_adjust_parallelism: plan.allow_adjust_parallelism, +// }); +// let fragment_type = match self.state { +// State::SelectLeaf => FragmentType::Source, +// State::MutationSource => FragmentType::MutationSource, +// State::Other => FragmentType::Intermediate, +// State::ReplaceInto => FragmentType::ReplaceInto, +// State::Compact => FragmentType::Compact, +// State::Recluster => FragmentType::Recluster, +// }; +// self.state = State::Other; +// let exchange = Self::get_exchange(self.ctx.clone(), &plan)?; +// +// let mut source_fragment = PlanFragment { +// plan, +// fragment_type, +// fragment_id: source_fragment_id, +// exchange, +// query_id: self.query_id.clone(), +// source_fragments: std::mem::take(&self.fragments), +// }; +// +// // Fill the destination_fragment_id for source fragments of `source_fragment`. +// Self::resolve_fragment_connection(&mut source_fragment); +// +// self.fragments.push(source_fragment); +// +// Ok(PhysicalPlan::ExchangeSource(ExchangeSource { +// // TODO(leiysky): we reuse the plan id here, +// // should generate a new one for the source. +// plan_id, +// +// schema: input_schema, +// query_id: self.query_id.clone(), +// +// source_fragment_id, +// })) +// } +// +// fn replace_union(&mut self, plan: &UnionAll) -> Result { +// let mut fragments = vec![]; +// let left_input = self.replace(plan.left.as_ref())?; +// let left_state = self.state.clone(); +// +// // Consume current fragments to prevent them being consumed by `right_input`. +// fragments.append(&mut self.fragments); +// let right_input = self.replace(plan.right.as_ref())?; +// let right_state = self.state.clone(); +// +// fragments.append(&mut self.fragments); +// self.fragments = fragments; +// +// // If any of the input is a source fragment, the union all is a source fragment. +// if left_state == State::SelectLeaf || right_state == State::SelectLeaf { +// self.state = State::SelectLeaf; +// } else { +// self.state = State::Other; +// } +// +// Ok(PhysicalPlan::UnionAll(UnionAll { +// left: Box::new(left_input), +// right: Box::new(right_input), +// ..plan.clone() +// })) +// } +// +// // TODO(Sky): remove redundant code +// fn replace_copy_into_table(&mut self, plan: &CopyIntoTable) -> Result { +// match &plan.source { +// CopyIntoTableSource::Stage(_) => { +// self.state = State::SelectLeaf; +// Ok(PhysicalPlan::CopyIntoTable(Box::new(plan.clone()))) +// } +// CopyIntoTableSource::Query(query_physical_plan) => { +// let input = self.replace(query_physical_plan)?; +// Ok(PhysicalPlan::CopyIntoTable(Box::new(CopyIntoTable { +// source: CopyIntoTableSource::Query(Box::new(input)), +// ..plan.clone() +// }))) +// } +// } +// } +// +// fn replace_compact_source(&mut self, plan: &CompactSource) -> Result { +// self.state = State::Compact; +// Ok(PhysicalPlan::CompactSource(Box::new(plan.clone()))) +// } +// +// fn replace_replace_into(&mut self, plan: &ReplaceInto) -> Result { +// let input = self.replace(&plan.input)?; +// self.state = State::ReplaceInto; +// Ok(PhysicalPlan::ReplaceInto(Box::new(ReplaceInto { +// input: Box::new(input), +// ..plan.clone() +// }))) +// } +// +// fn replace_mutation_source(&mut self, plan: &MutationSource) -> Result { +// self.state = State::MutationSource; +// Ok(PhysicalPlan::MutationSource(plan.clone())) +// } +// +// fn replace_mutation(&mut self, plan: &Mutation) -> Result { +// let input = self.replace(&plan.input)?; +// Ok(PhysicalPlan::Mutation(Box::new(Mutation { +// input: Box::new(input), +// ..plan.clone() +// }))) +// } +// } diff --git a/src/query/service/src/schedulers/fragments/plan_fragment.rs b/src/query/service/src/schedulers/fragments/plan_fragment.rs index f11d1812f3b37..b36ac62a9b495 100644 --- a/src/query/service/src/schedulers/fragments/plan_fragment.rs +++ b/src/query/service/src/schedulers/fragments/plan_fragment.rs @@ -28,13 +28,14 @@ use databend_common_sql::executor::physical_plans::CompactSource; use databend_common_sql::executor::physical_plans::ConstantTableScan; use databend_common_sql::executor::physical_plans::CopyIntoTable; use databend_common_sql::executor::physical_plans::CopyIntoTableSource; +use databend_common_sql::executor::physical_plans::ExchangeSink; use databend_common_sql::executor::physical_plans::MutationSource; use databend_common_sql::executor::physical_plans::Recluster; use databend_common_sql::executor::physical_plans::ReplaceDeduplicate; use databend_common_sql::executor::physical_plans::ReplaceInto; use databend_common_sql::executor::physical_plans::TableScan; +use databend_common_sql::executor::DeriveHandle; use databend_common_sql::executor::IPhysicalPlan; -use databend_common_sql::executor::PhysicalPlanDeriveHandle; use databend_common_storages_fuse::TableContext; use databend_storages_common_table_meta::meta::BlockSlotDescription; use databend_storages_common_table_meta::meta::Location; @@ -227,12 +228,11 @@ impl PlanFragment { ctx: Arc, fragment_actions: &mut QueryFragmentActions, ) -> Result<()> { - let plan = match &self.plan { - PhysicalPlan::ExchangeSink(plan) => plan, - _ => unreachable!("logic error"), + let Some(plan) = self.plan.down_cast::() else { + unreachable!("logic error"); }; - let plan = PhysicalPlan::ExchangeSink(plan.clone()); + let plan: Box = Box::new(plan.clone()); let mutation_source = plan.try_find_mutation_source().unwrap(); let partitions: &Partitions = &mutation_source.partitions; @@ -479,12 +479,12 @@ struct ReadSourceDeriveHandle { } impl ReadSourceDeriveHandle { - pub fn new(sources: HashMap) -> Box { + pub fn new(sources: HashMap) -> Box { Box::new(ReadSourceDeriveHandle { sources }) } } -impl PhysicalPlanDeriveHandle for ReadSourceDeriveHandle { +impl DeriveHandle for ReadSourceDeriveHandle { fn derive( &mut self, v: &Box, @@ -528,12 +528,12 @@ struct ReclusterDeriveHandle { } impl ReclusterDeriveHandle { - pub fn new(tasks: Vec) -> Box { + pub fn new(tasks: Vec) -> Box { Box::new(ReclusterDeriveHandle { tasks }) } } -impl PhysicalPlanDeriveHandle for ReclusterDeriveHandle { +impl DeriveHandle for ReclusterDeriveHandle { fn derive( &mut self, v: &Box, @@ -555,12 +555,12 @@ struct MutationSourceDeriveHandle { } impl MutationSourceDeriveHandle { - pub fn new(partitions: Partitions) -> Box { + pub fn new(partitions: Partitions) -> Box { Box::new(MutationSourceDeriveHandle { partitions }) } } -impl PhysicalPlanDeriveHandle for MutationSourceDeriveHandle { +impl DeriveHandle for MutationSourceDeriveHandle { fn derive( &mut self, v: &Box, @@ -582,12 +582,12 @@ struct CompactSourceDeriveHandle { } impl CompactSourceDeriveHandle { - pub fn new(partitions: Partitions) -> Box { + pub fn new(partitions: Partitions) -> Box { Box::new(CompactSourceDeriveHandle { partitions }) } } -impl PhysicalPlanDeriveHandle for CompactSourceDeriveHandle { +impl DeriveHandle for CompactSourceDeriveHandle { fn derive( &mut self, v: &Box, @@ -616,7 +616,7 @@ impl ReplaceDeriveHandle { partitions: Vec<(usize, Location)>, slot: Option, need_insert: bool, - ) -> Box { + ) -> Box { Box::new(ReplaceDeriveHandle { partitions, slot, @@ -625,7 +625,7 @@ impl ReplaceDeriveHandle { } } -impl PhysicalPlanDeriveHandle for ReplaceDeriveHandle { +impl DeriveHandle for ReplaceDeriveHandle { fn derive( &mut self, v: &Box, diff --git a/src/query/service/src/schedulers/fragments/query_fragment_actions.rs b/src/query/service/src/schedulers/fragments/query_fragment_actions.rs index ea2dccf3d2379..46ce59b5568c0 100644 --- a/src/query/service/src/schedulers/fragments/query_fragment_actions.rs +++ b/src/query/service/src/schedulers/fragments/query_fragment_actions.rs @@ -24,6 +24,7 @@ use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_expression::DataSchemaRef; use databend_common_meta_types::NodeInfo; +use databend_common_sql::executor::IPhysicalPlan; use crate::clusters::ClusterHelper; use crate::servers::flight::v1::exchange::DataExchange; @@ -38,15 +39,15 @@ use crate::sql::executor::PhysicalPlan; // Query plan fragment with executor name #[derive(Debug)] pub struct QueryFragmentAction { - pub physical_plan: PhysicalPlan, pub executor: String, + pub physical_plan: Box, } impl QueryFragmentAction { - pub fn create(executor: String, physical_plan: PhysicalPlan) -> QueryFragmentAction { + pub fn create(executor: String, physical_plan: Box) -> QueryFragmentAction { QueryFragmentAction { - physical_plan, executor, + physical_plan, } } } @@ -263,8 +264,8 @@ impl QueryFragmentsActions { for fragment_action in &fragment_actions.fragment_actions { let query_fragment = QueryFragment::create( fragment_actions.fragment_id, - fragment_action.physical_plan.clone(), fragment_actions.data_exchange.clone(), + fragment_action.physical_plan.clone(), ); match fragments_packets.entry(fragment_action.executor.clone()) { diff --git a/src/query/service/src/schedulers/scheduler.rs b/src/query/service/src/schedulers/scheduler.rs index d5ae57bc7776f..257737340ea10 100644 --- a/src/query/service/src/schedulers/scheduler.rs +++ b/src/query/service/src/schedulers/scheduler.rs @@ -18,6 +18,7 @@ use async_trait::async_trait; use databend_common_exception::Result; use databend_common_expression::DataBlock; use databend_common_sql::executor::build_broadcast_plans; +use databend_common_sql::executor::IPhysicalPlan; use databend_common_sql::planner::QueryExecutor; use databend_common_sql::Planner; use futures_util::TryStreamExt; @@ -61,7 +62,7 @@ pub async fn build_query_pipeline( #[async_backtrace::framed] pub async fn build_query_pipeline_without_render_result_set( ctx: &Arc, - plan: &PhysicalPlan, + plan: &Box, ) -> Result { let build_res = if !plan.is_distributed_plan() { build_local_pipeline(ctx, plan).await @@ -79,7 +80,7 @@ pub async fn build_query_pipeline_without_render_result_set( #[async_backtrace::framed] pub async fn build_local_pipeline( ctx: &Arc, - plan: &PhysicalPlan, + plan: &Box, ) -> Result { let pipeline = PipelineBuilder::create(ctx.get_function_context()?, ctx.get_settings(), ctx.clone()); @@ -94,7 +95,7 @@ pub async fn build_local_pipeline( #[async_backtrace::framed] pub async fn build_distributed_pipeline( ctx: &Arc, - plan: &PhysicalPlan, + plan: &Box, ) -> Result { let mut fragments_actions = QueryFragmentsActions::create(ctx.clone()); for plan in build_broadcast_plans(ctx.as_ref())? @@ -138,7 +139,7 @@ impl ServiceQueryExecutor { impl QueryExecutor for ServiceQueryExecutor { async fn execute_query_with_physical_plan( &self, - plan: &PhysicalPlan, + plan: &Box, ) -> Result> { let build_res = build_query_pipeline_without_render_result_set(&self.ctx, plan).await?; let settings = ExecutorSettings::try_create(self.ctx.clone())?; diff --git a/src/query/service/src/servers/flight/v1/packets/packet_fragment.rs b/src/query/service/src/servers/flight/v1/packets/packet_fragment.rs index c47877363d0a5..e3cee46023264 100644 --- a/src/query/service/src/servers/flight/v1/packets/packet_fragment.rs +++ b/src/query/service/src/servers/flight/v1/packets/packet_fragment.rs @@ -15,21 +15,23 @@ use std::fmt::Debug; use std::fmt::Formatter; +use databend_common_sql::executor::IPhysicalPlan; + use crate::servers::flight::v1::exchange::DataExchange; use crate::sql::executor::PhysicalPlan; #[derive(Clone, serde::Serialize, serde::Deserialize)] pub struct QueryFragment { - pub physical_plan: PhysicalPlan, pub fragment_id: usize, + pub physical_plan: Box, pub data_exchange: Option, } impl QueryFragment { pub fn create( fragment_id: usize, - physical_plan: PhysicalPlan, data_exchange: Option, + physical_plan: Box, ) -> QueryFragment { QueryFragment { physical_plan, diff --git a/src/query/sql/src/executor/format.rs b/src/query/sql/src/executor/format.rs index 684607c500e4e..8b69f1c713ea7 100644 --- a/src/query/sql/src/executor/format.rs +++ b/src/query/sql/src/executor/format.rs @@ -73,461 +73,9 @@ use crate::planner::DUMMY_TABLE_INDEX; use crate::plans::CacheSource; use crate::IndexType; -impl PhysicalPlan { - pub fn format( - &self, - metadata: MetadataRef, - profs: HashMap, - ) -> Result> { - let metadata = metadata.read().clone(); - let mut context = FormatContext { - scan_id_to_runtime_filters: HashMap::new(), - }; - to_format_tree(self, &metadata, &profs, &mut context) - } - - #[recursive::recursive] - pub fn format_join(&self, metadata: &MetadataRef) -> Result> { - match self { - PhysicalPlan::TableScan(plan) => { - if plan.table_index == Some(DUMMY_TABLE_INDEX) { - return Ok(FormatTreeNode::with_children( - format!("Scan: dummy, rows: {}", plan.source.statistics.read_rows), - vec![], - )); - } - - match plan.table_index { - None => Ok(FormatTreeNode::with_children( - format!( - "Scan: {}.{} (read rows: {})", - plan.source.source_info.catalog_name(), - plan.source.source_info.desc(), - plan.source.statistics.read_rows - ), - vec![], - )), - Some(table_index) => { - let table = metadata.read().table(table_index).clone(); - let table_name = - format!("{}.{}.{}", table.catalog(), table.database(), table.name()); - - Ok(FormatTreeNode::with_children( - format!( - "Scan: {} (#{}) (read rows: {})", - table_name, table_index, plan.source.statistics.read_rows - ), - vec![], - )) - } - } - } - PhysicalPlan::HashJoin(plan) => { - let build_child = plan.build.format_join(metadata)?; - let probe_child = plan.probe.format_join(metadata)?; - - let children = vec![ - FormatTreeNode::with_children("Build".to_string(), vec![build_child]), - FormatTreeNode::with_children("Probe".to_string(), vec![probe_child]), - ]; - - let _estimated_rows = if let Some(info) = &plan.stat_info { - format!("{0:.2}", info.estimated_rows) - } else { - String::from("None") - }; - - Ok(FormatTreeNode::with_children( - format!("HashJoin: {}", plan.join_type), - children, - )) - } - PhysicalPlan::RangeJoin(plan) => { - let left_child = plan.left.format_join(metadata)?; - let right_child = plan.right.format_join(metadata)?; - - let children = vec![ - FormatTreeNode::with_children("Left".to_string(), vec![left_child]), - FormatTreeNode::with_children("Right".to_string(), vec![right_child]), - ]; - - let _estimated_rows = if let Some(info) = &plan.stat_info { - format!("{0:.2}", info.estimated_rows) - } else { - String::from("none") - }; - - Ok(FormatTreeNode::with_children( - format!("RangeJoin: {}", plan.join_type), - children, - )) - } - PhysicalPlan::UnionAll(union_all) => { - let left_child = union_all.left.format_join(metadata)?; - let right_child = union_all.right.format_join(metadata)?; - - let children = vec![ - FormatTreeNode::with_children("Left".to_string(), vec![left_child]), - FormatTreeNode::with_children("Right".to_string(), vec![right_child]), - ]; - - Ok(FormatTreeNode::with_children( - "UnionAll".to_string(), - children, - )) - } - other => { - let children = other - .children() - .map(|child| child.format_join(metadata)) - .collect::>>>()?; - - if children.len() == 1 { - Ok(children[0].clone()) - } else { - Ok(FormatTreeNode::with_children( - format!("{:?}", other), - children, - )) - } - } - } - } -} - -// The method will only collect scan,filter and join nodes -// It's only used to debug cardinality estimator. -#[recursive::recursive] -pub fn format_partial_tree( - plan: &PhysicalPlan, - metadata: &MetadataRef, - profs: &HashMap, -) -> Result> { - match plan { - PhysicalPlan::TableScan(plan) => { - if plan.table_index == Some(DUMMY_TABLE_INDEX) { - return Ok(FormatTreeNode::new("DummyTableScan".to_string())); - } - let table_name = match plan.table_index { - None => format!( - "{}.{}", - plan.source.source_info.catalog_name(), - plan.source.source_info.desc() - ), - Some(table_index) => { - let metadata = metadata.read().clone(); - let table = metadata.table(table_index).clone(); - format!("{}.{}.{}", table.catalog(), table.database(), table.name()) - } - }; - let mut children = vec![FormatTreeNode::new(format!("table: {table_name}"))]; - if let Some(info) = &plan.stat_info { - let items = plan_stats_info_to_format_tree(info); - children.extend(items); - } - append_output_rows_info(&mut children, profs, plan.plan_id); - - Ok(FormatTreeNode::with_children( - "TableScan".to_string(), - children, - )) - } - PhysicalPlan::Filter(plan) => { - let filter = plan - .predicates - .iter() - .map(|pred| pred.as_expr(&BUILTIN_FUNCTIONS).sql_display()) - .join(", "); - let mut children = vec![FormatTreeNode::new(format!("filters: [{filter}]"))]; - if let Some(info) = &plan.stat_info { - let items = plan_stats_info_to_format_tree(info); - children.extend(items); - } - append_output_rows_info(&mut children, profs, plan.plan_id); - children.push(format_partial_tree(&plan.input, metadata, profs)?); - Ok(FormatTreeNode::with_children( - "Filter".to_string(), - children, - )) - } - PhysicalPlan::HashJoin(plan) => { - let build_child = format_partial_tree(&plan.build, metadata, profs)?; - let probe_child = format_partial_tree(&plan.probe, metadata, profs)?; - let mut children = vec![]; - if let Some(info) = &plan.stat_info { - let items = plan_stats_info_to_format_tree(info); - children.extend(items); - } - append_output_rows_info(&mut children, profs, plan.plan_id); - children.push(build_child); - children.push(probe_child); - - Ok(FormatTreeNode::with_children( - format!("HashJoin: {}", plan.join_type), - children, - )) - } - PhysicalPlan::RangeJoin(plan) => { - let left_child = format_partial_tree(&plan.left, metadata, profs)?; - let right_child = format_partial_tree(&plan.right, metadata, profs)?; - let mut children = vec![]; - if let Some(info) = &plan.stat_info { - let items = plan_stats_info_to_format_tree(info); - children.extend(items); - } - append_output_rows_info(&mut children, profs, plan.plan_id); - - let children = vec![ - FormatTreeNode::with_children("Left".to_string(), vec![left_child]), - FormatTreeNode::with_children("Right".to_string(), vec![right_child]), - ]; - - Ok(FormatTreeNode::with_children( - format!("RangeJoin: {}", plan.join_type), - children, - )) - } - PhysicalPlan::UnionAll(union_all) => { - let left_child = format_partial_tree(&union_all.left, metadata, profs)?; - let right_child = format_partial_tree(&union_all.right, metadata, profs)?; - let mut children = vec![]; - if let Some(info) = &union_all.stat_info { - let items = plan_stats_info_to_format_tree(info); - children.extend(items); - } - append_output_rows_info(&mut children, profs, union_all.plan_id); - let children = vec![ - FormatTreeNode::with_children("Left".to_string(), vec![left_child]), - FormatTreeNode::with_children("Right".to_string(), vec![right_child]), - ]; - - Ok(FormatTreeNode::with_children( - "UnionAll".to_string(), - children, - )) - } - PhysicalPlan::MutationSource(plan) => { - let metadata = metadata.read().clone(); - let table = metadata.table(plan.table_index).clone(); - let table_name = format!("{}.{}.{}", table.catalog(), table.database(), table.name()); - let mut children = vec![FormatTreeNode::new(format!("table: {table_name}"))]; - if let Some(filters) = &plan.filters { - let filter = filters.filter.as_expr(&BUILTIN_FUNCTIONS).sql_display(); - children.push(FormatTreeNode::new(format!("filters: [{filter}]"))); - } - append_output_rows_info(&mut children, profs, plan.plan_id); - Ok(FormatTreeNode::with_children( - "MutationSource".to_string(), - children, - )) - } - other => { - let children = other - .children() - .map(|child| format_partial_tree(child, metadata, profs)) - .collect::>>>()?; - - if children.len() == 1 { - Ok(children[0].clone()) - } else { - Ok(FormatTreeNode::with_children( - format!("{:?}", other), - children, - )) - } - } - } -} - -struct FormatContext { - scan_id_to_runtime_filters: HashMap>, -} - -#[recursive::recursive] -fn to_format_tree( - plan: &PhysicalPlan, - metadata: &Metadata, - profs: &HashMap, - context: &mut FormatContext, -) -> Result> { - match plan { - PhysicalPlan::TableScan(plan) => table_scan_to_format_tree(plan, metadata, profs, context), - PhysicalPlan::Filter(plan) => filter_to_format_tree(plan, metadata, profs, context), - PhysicalPlan::EvalScalar(plan) => { - eval_scalar_to_format_tree(plan, metadata, profs, context) - } - PhysicalPlan::AggregateExpand(plan) => { - aggregate_expand_to_format_tree(plan, metadata, profs, context) - } - PhysicalPlan::AggregatePartial(plan) => { - aggregate_partial_to_format_tree(plan, metadata, profs, context) - } - PhysicalPlan::AggregateFinal(plan) => { - aggregate_final_to_format_tree(plan, metadata, profs, context) - } - PhysicalPlan::Window(plan) => window_to_format_tree(plan, metadata, profs, context), - PhysicalPlan::WindowPartition(plan) => { - window_partition_to_format_tree(plan, metadata, profs, context) - } - PhysicalPlan::Sort(plan) => sort_to_format_tree(plan, metadata, profs, context), - PhysicalPlan::Limit(plan) => limit_to_format_tree(plan, metadata, profs, context), - PhysicalPlan::RowFetch(plan) => row_fetch_to_format_tree(plan, metadata, profs, context), - PhysicalPlan::HashJoin(plan) => hash_join_to_format_tree(plan, metadata, profs, context), - PhysicalPlan::Exchange(plan) => exchange_to_format_tree(plan, metadata, profs, context), - PhysicalPlan::UnionAll(plan) => union_all_to_format_tree(plan, metadata, profs, context), - PhysicalPlan::ExchangeSource(plan) => exchange_source_to_format_tree(plan, metadata), - PhysicalPlan::ExchangeSink(plan) => { - exchange_sink_to_format_tree(plan, metadata, profs, context) - } - PhysicalPlan::DistributedInsertSelect(plan) => { - distributed_insert_to_format_tree(plan.as_ref(), metadata, profs, context) - } - PhysicalPlan::Recluster(_) => Ok(FormatTreeNode::new("Recluster".to_string())), - PhysicalPlan::HilbertPartition(_) => { - Ok(FormatTreeNode::new("HilbertPartition".to_string())) - } - PhysicalPlan::CompactSource(_) => Ok(FormatTreeNode::new("CompactSource".to_string())), - PhysicalPlan::CommitSink(plan) => { - commit_sink_to_format_tree(plan, metadata, profs, context) - } - PhysicalPlan::ProjectSet(plan) => { - project_set_to_format_tree(plan, metadata, profs, context) - } - PhysicalPlan::Udf(plan) => udf_to_format_tree(plan, metadata, profs, context), - PhysicalPlan::RangeJoin(plan) => range_join_to_format_tree(plan, metadata, profs, context), - PhysicalPlan::CopyIntoTable(plan) => copy_into_table(plan), - PhysicalPlan::CopyIntoLocation(plan) => copy_into_location(plan), - PhysicalPlan::ReplaceAsyncSourcer(_) => { - Ok(FormatTreeNode::new("ReplaceAsyncSourcer".to_string())) - } - PhysicalPlan::ReplaceDeduplicate(_) => { - Ok(FormatTreeNode::new("ReplaceDeduplicate".to_string())) - } - PhysicalPlan::ReplaceInto(_) => Ok(FormatTreeNode::new("Replace".to_string())), - PhysicalPlan::MutationSource(plan) => format_mutation_source(plan, metadata, profs), - PhysicalPlan::ColumnMutation(plan) => { - format_column_mutation(plan, metadata, profs, context) - } - PhysicalPlan::Mutation(plan) => format_merge_into(plan, metadata, profs, context), - PhysicalPlan::MutationSplit(plan) => { - format_merge_into_split(plan, metadata, profs, context) - } - PhysicalPlan::MutationManipulate(plan) => { - format_merge_into_manipulate(plan, metadata, profs, context) - } - PhysicalPlan::MutationOrganize(plan) => { - format_merge_into_organize(plan, metadata, profs, context) - } - PhysicalPlan::AddStreamColumn(plan) => { - format_add_stream_column(plan, metadata, profs, context) - } - PhysicalPlan::RecursiveCteScan(_) => { - Ok(FormatTreeNode::new("RecursiveCTEScan".to_string())) - } - PhysicalPlan::ConstantTableScan(plan) => constant_table_scan_to_format_tree(plan, metadata), - PhysicalPlan::ExpressionScan(plan) => { - expression_scan_to_format_tree(plan, metadata, profs, context) - } - PhysicalPlan::CacheScan(plan) => cache_scan_to_format_tree(plan, metadata), - PhysicalPlan::Duplicate(plan) => { - let mut children = Vec::new(); - children.push(FormatTreeNode::new(format!( - "Duplicate data to {} branch", - plan.n - ))); - append_profile_info(&mut children, profs, plan.plan_id); - children.push(to_format_tree(&plan.input, metadata, profs, context)?); - Ok(FormatTreeNode::with_children( - "Duplicate".to_string(), - children, - )) - } - PhysicalPlan::Shuffle(plan) => to_format_tree(&plan.input, metadata, profs, context), /* will be hided in explain */ - PhysicalPlan::ChunkFilter(plan) => { - if plan.predicates.iter().all(|x| x.is_none()) { - return to_format_tree(&plan.input, metadata, profs, context); - } - let mut children = Vec::new(); - for (i, predicate) in plan.predicates.iter().enumerate() { - if let Some(predicate) = predicate { - children.push(FormatTreeNode::new(format!( - "branch {}: {}", - i, - predicate.as_expr(&BUILTIN_FUNCTIONS).sql_display() - ))); - } else { - children.push(FormatTreeNode::new(format!("branch {}: None", i))); - } - } - append_profile_info(&mut children, profs, plan.plan_id); - children.push(to_format_tree(&plan.input, metadata, profs, context)?); - Ok(FormatTreeNode::with_children( - "Filter".to_string(), - children, - )) - } - PhysicalPlan::ChunkEvalScalar(plan) => { - let mut children = Vec::new(); - if plan.eval_scalars.iter().all(|x| x.is_none()) { - return to_format_tree(&plan.input, metadata, profs, context); - } - for (i, eval_scalar) in plan.eval_scalars.iter().enumerate() { - if let Some(eval_scalar) = eval_scalar { - children.push(FormatTreeNode::new(format!( - "branch {}: {}", - i, - eval_scalar - .remote_exprs - .iter() - .map(|x| x.as_expr(&BUILTIN_FUNCTIONS).sql_display()) - .join(", ") - ))); - } else { - children.push(FormatTreeNode::new(format!("branch {}: None", i))); - } - } - append_profile_info(&mut children, profs, plan.plan_id); - children.push(to_format_tree(&plan.input, metadata, profs, context)?); - Ok(FormatTreeNode::with_children( - "EvalScalar".to_string(), - children, - )) - } - PhysicalPlan::ChunkCastSchema(plan) => { - to_format_tree(&plan.input, metadata, profs, context) - } // will be hided in explain - PhysicalPlan::ChunkFillAndReorder(plan) => { - to_format_tree(&plan.input, metadata, profs, context) - } // will be hided in explain - PhysicalPlan::ChunkAppendData(plan) => { - let mut children = Vec::new(); - append_profile_info(&mut children, profs, plan.plan_id); - children.push(to_format_tree(&plan.input, metadata, profs, context)?); - Ok(FormatTreeNode::with_children( - "WriteData".to_string(), - children, - )) - } - PhysicalPlan::ChunkMerge(plan) => to_format_tree(&plan.input, metadata, profs, context), /* will be hided in explain */ - PhysicalPlan::ChunkCommitInsert(plan) => { - let mut children = Vec::new(); - append_profile_info(&mut children, profs, plan.plan_id); - children.push(to_format_tree(&plan.input, metadata, profs, context)?); - Ok(FormatTreeNode::with_children( - "Commit".to_string(), - children, - )) - } - PhysicalPlan::AsyncFunction(plan) => { - async_function_to_format_tree(plan, metadata, profs, context) - } - PhysicalPlan::BroadcastSource(_plan) => { - Ok(FormatTreeNode::new("RuntimeFilterSource".to_string())) - } - PhysicalPlan::BroadcastSink(_plan) => { - Ok(FormatTreeNode::new("RuntimeFilterSink".to_string())) - } - } +pub struct FormatContext<'a> { + pub metadata: &'a Metadata, + pub scan_id_to_runtime_filters: HashMap>, } /// Helper function to add profile info to the format tree. @@ -571,525 +119,6 @@ fn append_output_rows_info( } } -fn format_mutation_source( - plan: &MutationSource, - metadata: &Metadata, - profs: &HashMap, -) -> Result> { - let table = metadata.table(plan.table_index); - let table_name = format!("{}.{}.{}", table.catalog(), table.database(), table.name()); - let filters = plan - .filters - .as_ref() - .map(|filters| filters.filter.as_expr(&BUILTIN_FUNCTIONS).sql_display()) - .unwrap_or_default(); - let mut children = vec![ - FormatTreeNode::new(format!("table: {table_name}")), - FormatTreeNode::new(format!( - "output columns: [{}]", - format_output_columns(plan.output_schema()?, metadata, false) - )), - FormatTreeNode::new(format!("filters: [{filters}]")), - ]; - - let payload = match plan.input_type { - MutationType::Update => "Update", - MutationType::Delete => { - if plan.truncate_table { - "DeleteAll" - } else { - "Delete" - } - } - MutationType::Merge => "Merge", - }; - - // Part stats. - children.extend(part_stats_info_to_format_tree(&plan.statistics)); - append_profile_info(&mut children, profs, plan.plan_id); - - Ok(FormatTreeNode::with_children( - format!("MutationSource({})", payload), - children, - )) -} - -fn format_column_mutation( - plan: &ColumnMutation, - metadata: &Metadata, - profs: &HashMap, - context: &mut FormatContext, -) -> Result> { - to_format_tree(&plan.input, metadata, profs, context) -} - -fn format_merge_into( - merge_into: &Mutation, - metadata: &Metadata, - profs: &HashMap, - context: &mut FormatContext, -) -> Result> { - let table_entry = metadata.table(merge_into.target_table_index).clone(); - let target_table = vec![FormatTreeNode::new(format!( - "target table: [catalog: {}] [database: {}] [table: {}]", - table_entry.catalog(), - table_entry.database(), - table_entry.name() - ))]; - let target_schema = table_entry.table().schema_with_stream(); - - let merge_into_organize: &PhysicalPlan = &merge_into.input; - let merge_into_manipulate: &PhysicalPlan = - if let PhysicalPlan::MutationOrganize(plan) = merge_into_organize { - &plan.input - } else { - return Err(ErrorCode::Internal( - "Expect MutationOrganize after MergeIntoSerialize ".to_string(), - )); - }; - - let children = if let PhysicalPlan::MutationManipulate(plan) = merge_into_manipulate { - // Matched clauses. - let mut matched_children = Vec::with_capacity(plan.matched.len()); - for evaluator in &plan.matched { - let condition_format = evaluator.0.as_ref().map_or_else( - || "condition: None".to_string(), - |predicate| { - format!( - "condition: {}", - predicate.as_expr(&BUILTIN_FUNCTIONS).sql_display() - ) - }, - ); - if evaluator.1.is_none() { - matched_children.push(FormatTreeNode::new(format!( - "matched delete: [{}]", - condition_format - ))); - } else { - let mut update_list = evaluator.1.as_ref().unwrap().clone(); - update_list.sort_by(|a, b| a.0.cmp(&b.0)); - let update_format = update_list - .iter() - .map(|(field_idx, expr)| { - format!( - "{} = {}", - target_schema.field(*field_idx).name(), - expr.as_expr(&BUILTIN_FUNCTIONS).sql_display() - ) - }) - .join(","); - matched_children.push(FormatTreeNode::new(format!( - "matched update: [{}, update set {}]", - condition_format, update_format - ))); - } - } - - // UnMatched clauses. - let mut unmatched_children = Vec::with_capacity(plan.unmatched.len()); - for evaluator in &plan.unmatched { - let condition_format = evaluator.1.as_ref().map_or_else( - || "condition: None".to_string(), - |predicate| { - format!( - "condition: {}", - predicate.as_expr(&BUILTIN_FUNCTIONS).sql_display() - ) - }, - ); - let insert_schema_format = evaluator - .0 - .fields - .iter() - .map(|field| field.name()) - .join(","); - let values_format = evaluator - .2 - .iter() - .map(|expr| expr.as_expr(&BUILTIN_FUNCTIONS).sql_display()) - .join(","); - let unmatched_format = format!( - "insert into ({}) values({})", - insert_schema_format, values_format - ); - unmatched_children.push(FormatTreeNode::new(format!( - "unmatched insert: [{}, {}]", - condition_format, unmatched_format - ))); - } - - [target_table, matched_children, unmatched_children, vec![ - to_format_tree(&plan.input, metadata, profs, context)?, - ]] - .concat() - } else { - return Err(ErrorCode::Internal( - "Expect MutationManipulate after MutationOrganize ".to_string(), - )); - }; - Ok(FormatTreeNode::with_children( - "DataMutation".to_string(), - children, - )) -} - -fn format_merge_into_split( - plan: &MutationSplit, - metadata: &Metadata, - profs: &HashMap, - context: &mut FormatContext, -) -> Result> { - to_format_tree(&plan.input, metadata, profs, context) -} - -fn format_merge_into_manipulate( - plan: &MutationManipulate, - metadata: &Metadata, - profs: &HashMap, - context: &mut FormatContext, -) -> Result> { - to_format_tree(&plan.input, metadata, profs, context) -} - -fn format_merge_into_organize( - plan: &MutationOrganize, - metadata: &Metadata, - profs: &HashMap, - context: &mut FormatContext, -) -> Result> { - to_format_tree(&plan.input, metadata, profs, context) -} - -fn format_add_stream_column( - plan: &AddStreamColumn, - metadata: &Metadata, - profs: &HashMap, - context: &mut FormatContext, -) -> Result> { - to_format_tree(&plan.input, metadata, profs, context) -} - -fn copy_into_table(plan: &CopyIntoTable) -> Result> { - Ok(FormatTreeNode::new(format!( - "CopyIntoTable: {}", - plan.table_info - ))) -} - -fn copy_into_location(_: &CopyIntoLocation) -> Result> { - Ok(FormatTreeNode::new("CopyIntoLocation".to_string())) -} - -fn table_scan_to_format_tree( - plan: &TableScan, - metadata: &Metadata, - profs: &HashMap, - context: &mut FormatContext, -) -> Result> { - if plan.table_index == Some(DUMMY_TABLE_INDEX) { - return Ok(FormatTreeNode::new("DummyTableScan".to_string())); - } - - let table_name = match plan.table_index { - None => format!( - "{}.{}", - plan.source.source_info.catalog_name(), - plan.source.source_info.desc() - ), - Some(table_index) => { - let table = metadata.table(table_index).clone(); - format!("{}.{}.{}", table.catalog(), table.database(), table.name()) - } - }; - let filters = plan - .source - .push_downs - .as_ref() - .and_then(|extras| { - extras - .filters - .as_ref() - .map(|filters| filters.filter.as_expr(&BUILTIN_FUNCTIONS).sql_display()) - }) - .unwrap_or_default(); - - let limit = plan - .source - .push_downs - .as_ref() - .map_or("NONE".to_string(), |extras| { - extras - .limit - .map_or("NONE".to_string(), |limit| limit.to_string()) - }); - - let virtual_columns = plan.source.push_downs.as_ref().and_then(|extras| { - extras.virtual_column.as_ref().map(|virtual_column| { - let mut names = virtual_column - .virtual_column_fields - .iter() - .map(|c| c.name.clone()) - .collect::>(); - names.sort(); - names.iter().join(", ") - }) - }); - - let agg_index = plan - .source - .push_downs - .as_ref() - .and_then(|extras| extras.agg_index.as_ref()); - - let mut children = vec![ - FormatTreeNode::new(format!("table: {table_name}")), - FormatTreeNode::new(format!( - "output columns: [{}]", - format_output_columns(plan.output_schema()?, metadata, false) - )), - ]; - - // Part stats. - children.extend(part_stats_info_to_format_tree(&plan.source.statistics)); - // Push downs. - let push_downs = format!("push downs: [filters: [{filters}], limit: {limit}]"); - children.push(FormatTreeNode::new(push_downs)); - - // runtime filters - let rf = context.scan_id_to_runtime_filters.get(&plan.scan_id); - if let Some(rf) = rf { - let rf = rf.iter().map(|rf| format!("#{:?}", rf.id)).join(", "); - children.push(FormatTreeNode::new(format!("apply join filters: [{rf}]"))); - } - - // Virtual columns. - if let Some(virtual_columns) = virtual_columns { - if !virtual_columns.is_empty() { - let virtual_columns = format!("virtual columns: [{virtual_columns}]"); - children.push(FormatTreeNode::new(virtual_columns)); - } - } - - // Aggregating index - if let Some(agg_index) = agg_index { - let (_, agg_index_sql, _) = metadata - .get_agg_indexes(&table_name) - .unwrap() - .iter() - .find(|(index, _, _)| *index == agg_index.index_id) - .unwrap(); - - children.push(FormatTreeNode::new(format!( - "aggregating index: [{agg_index_sql}]" - ))); - - let agg_sel = agg_index - .selection - .iter() - .map(|(expr, _)| expr.as_expr(&BUILTIN_FUNCTIONS).sql_display()) - .join(", "); - let agg_filter = agg_index - .filter - .as_ref() - .map(|f| f.as_expr(&BUILTIN_FUNCTIONS).sql_display()); - let text = if let Some(f) = agg_filter { - format!("rewritten query: [selection: [{agg_sel}], filter: {f}]") - } else { - format!("rewritten query: [selection: [{agg_sel}]]") - }; - children.push(FormatTreeNode::new(text)); - } - - if let Some(info) = &plan.stat_info { - let items = plan_stats_info_to_format_tree(info); - children.extend(items); - } - - append_profile_info(&mut children, profs, plan.plan_id); - - Ok(FormatTreeNode::with_children( - "TableScan".to_string(), - children, - )) -} - -fn constant_table_scan_to_format_tree( - plan: &ConstantTableScan, - metadata: &Metadata, -) -> Result> { - if plan.num_rows == 0 { - return Ok(FormatTreeNode::new(plan.name().to_string())); - } - - let mut children = Vec::with_capacity(plan.values.len() + 1); - children.push(FormatTreeNode::new(format!( - "output columns: [{}]", - format_output_columns(plan.output_schema()?, metadata, true) - ))); - for (i, value) in plan.values.iter().enumerate() { - let column = value.iter().map(|val| format!("{val}")).join(", "); - children.push(FormatTreeNode::new(format!("column {}: [{}]", i, column))); - } - Ok(FormatTreeNode::with_children( - plan.name().to_string(), - children, - )) -} - -fn expression_scan_to_format_tree( - plan: &ExpressionScan, - metadata: &Metadata, - profs: &HashMap, - context: &mut FormatContext, -) -> Result> { - let mut children = Vec::with_capacity(plan.values.len() + 1); - children.push(FormatTreeNode::new(format!( - "output columns: [{}]", - format_output_columns(plan.output_schema()?, metadata, true) - ))); - for (i, value) in plan.values.iter().enumerate() { - let column = value - .iter() - .map(|val| val.as_expr(&BUILTIN_FUNCTIONS).sql_display()) - .join(", "); - children.push(FormatTreeNode::new(format!("column {}: [{}]", i, column))); - } - - children.push(to_format_tree(&plan.input, metadata, profs, context)?); - - Ok(FormatTreeNode::with_children( - "ExpressionScan".to_string(), - children, - )) -} - -fn cache_scan_to_format_tree( - plan: &CacheScan, - metadata: &Metadata, -) -> Result> { - let mut children = Vec::with_capacity(2); - children.push(FormatTreeNode::new(format!( - "output columns: [{}]", - format_output_columns(plan.output_schema()?, metadata, true) - ))); - - match &plan.cache_source { - CacheSource::HashJoinBuild((cache_index, column_indexes)) => { - let mut column_indexes = column_indexes.clone(); - column_indexes.sort(); - children.push(FormatTreeNode::new(format!("cache index: {}", cache_index))); - children.push(FormatTreeNode::new(format!( - "column indexes: {:?}", - column_indexes - ))); - } - } - - Ok(FormatTreeNode::with_children( - "CacheScan".to_string(), - children, - )) -} - -fn filter_to_format_tree( - plan: &Filter, - metadata: &Metadata, - profs: &HashMap, - context: &mut FormatContext, -) -> Result> { - let filter = plan - .predicates - .iter() - .map(|pred| pred.as_expr(&BUILTIN_FUNCTIONS).sql_display()) - .join(", "); - let mut children = vec![ - FormatTreeNode::new(format!( - "output columns: [{}]", - format_output_columns(plan.output_schema()?, metadata, true) - )), - FormatTreeNode::new(format!("filters: [{filter}]")), - ]; - - if let Some(info) = &plan.stat_info { - let items = plan_stats_info_to_format_tree(info); - children.extend(items); - } - - append_profile_info(&mut children, profs, plan.plan_id); - - children.push(to_format_tree(&plan.input, metadata, profs, context)?); - - Ok(FormatTreeNode::with_children( - "Filter".to_string(), - children, - )) -} - -fn eval_scalar_to_format_tree( - plan: &EvalScalar, - metadata: &Metadata, - profs: &HashMap, - context: &mut FormatContext, -) -> Result> { - if plan.exprs.is_empty() { - return to_format_tree(&plan.input, metadata, profs, context); - } - let scalars = plan - .exprs - .iter() - .map(|(expr, _)| expr.as_expr(&BUILTIN_FUNCTIONS).sql_display()) - .collect::>() - .join(", "); - let mut children = vec![ - FormatTreeNode::new(format!( - "output columns: [{}]", - format_output_columns(plan.output_schema()?, metadata, true) - )), - FormatTreeNode::new(format!("expressions: [{scalars}]")), - ]; - - if let Some(info) = &plan.stat_info { - let items = plan_stats_info_to_format_tree(info); - children.extend(items); - } - - append_profile_info(&mut children, profs, plan.plan_id); - - children.push(to_format_tree(&plan.input, metadata, profs, context)?); - - Ok(FormatTreeNode::with_children( - "EvalScalar".to_string(), - children, - )) -} - -fn async_function_to_format_tree( - plan: &AsyncFunction, - metadata: &Metadata, - profs: &HashMap, - context: &mut FormatContext, -) -> Result> { - let mut children = vec![FormatTreeNode::new(format!( - "output columns: [{}]", - format_output_columns(plan.output_schema()?, metadata, true) - ))]; - - if let Some(info) = &plan.stat_info { - let items = plan_stats_info_to_format_tree(info); - children.extend(items); - } - - append_profile_info(&mut children, profs, plan.plan_id); - - children.push(to_format_tree(&plan.input, metadata, profs, context)?); - - Ok(FormatTreeNode::with_children( - "AsyncFunction".to_string(), - children, - )) -} - pub fn pretty_display_agg_desc(desc: &AggregateFunctionDesc, metadata: &Metadata) -> String { format!( "{}({})", @@ -1102,594 +131,7 @@ pub fn pretty_display_agg_desc(desc: &AggregateFunctionDesc, metadata: &Metadata ) } -fn aggregate_expand_to_format_tree( - plan: &AggregateExpand, - metadata: &Metadata, - profs: &HashMap, - context: &mut FormatContext, -) -> Result> { - let sets = plan - .grouping_sets - .sets - .iter() - .map(|set| { - set.iter() - .map(|&index| metadata.column(index).name()) - .collect::>() - .join(", ") - }) - .map(|s| format!("({})", s)) - .collect::>() - .join(", "); - - let mut children = vec![ - FormatTreeNode::new(format!( - "output columns: [{}]", - format_output_columns(plan.output_schema()?, metadata, true) - )), - FormatTreeNode::new(format!("grouping sets: [{sets}]")), - ]; - - if let Some(info) = &plan.stat_info { - let items = plan_stats_info_to_format_tree(info); - children.extend(items); - } - - append_profile_info(&mut children, profs, plan.plan_id); - - children.push(to_format_tree(&plan.input, metadata, profs, context)?); - - Ok(FormatTreeNode::with_children( - "AggregateExpand".to_string(), - children, - )) -} - -fn aggregate_partial_to_format_tree( - plan: &AggregatePartial, - metadata: &Metadata, - profs: &HashMap, - context: &mut FormatContext, -) -> Result> { - let group_by = plan - .group_by - .iter() - .map(|&index| metadata.column(index).name()) - .join(", "); - let agg_funcs = plan - .agg_funcs - .iter() - .map(|agg| pretty_display_agg_desc(agg, metadata)) - .collect::>() - .join(", "); - - let mut children = vec![ - FormatTreeNode::new(format!("group by: [{group_by}]")), - FormatTreeNode::new(format!("aggregate functions: [{agg_funcs}]")), - ]; - - if let Some(info) = &plan.stat_info { - let items = plan_stats_info_to_format_tree(info); - children.extend(items); - } - - if let Some((_, r)) = &plan.rank_limit { - children.push(FormatTreeNode::new(format!("rank limit: {r}"))); - } - - append_profile_info(&mut children, profs, plan.plan_id); - - children.push(to_format_tree(&plan.input, metadata, profs, context)?); - - Ok(FormatTreeNode::with_children( - "AggregatePartial".to_string(), - children, - )) -} - -fn aggregate_final_to_format_tree( - plan: &AggregateFinal, - metadata: &Metadata, - profs: &HashMap, - context: &mut FormatContext, -) -> Result> { - let group_by = plan - .group_by - .iter() - .map(|&index| { - let name = metadata.column(index).name(); - Ok(name) - }) - .collect::>>()? - .join(", "); - - let agg_funcs = plan - .agg_funcs - .iter() - .map(|agg| pretty_display_agg_desc(agg, metadata)) - .collect::>() - .join(", "); - - let mut children = vec![ - FormatTreeNode::new(format!( - "output columns: [{}]", - format_output_columns(plan.output_schema()?, metadata, true) - )), - FormatTreeNode::new(format!("group by: [{group_by}]")), - FormatTreeNode::new(format!("aggregate functions: [{agg_funcs}]")), - ]; - - if let Some(info) = &plan.stat_info { - let items = plan_stats_info_to_format_tree(info); - children.extend(items); - } - - append_profile_info(&mut children, profs, plan.plan_id); - - children.push(to_format_tree(&plan.input, metadata, profs, context)?); - - Ok(FormatTreeNode::with_children( - "AggregateFinal".to_string(), - children, - )) -} - -fn window_to_format_tree( - plan: &Window, - metadata: &Metadata, - profs: &HashMap, - context: &mut FormatContext, -) -> Result> { - let partition_by = plan - .partition_by - .iter() - .map(|&index| { - let name = metadata.column(index).name(); - Ok(name) - }) - .collect::>>()? - .join(", "); - - let order_by = plan - .order_by - .iter() - .map(|v| v.display_name.clone()) - .collect::>() - .join(", "); - - let frame = plan.window_frame.to_string(); - - let func = match &plan.func { - WindowFunction::Aggregate(agg) => pretty_display_agg_desc(agg, metadata), - func => format!("{}", func), - }; - - let mut children = vec![ - FormatTreeNode::new(format!( - "output columns: [{}]", - format_output_columns(plan.output_schema()?, metadata, true) - )), - FormatTreeNode::new(format!("aggregate function: [{func}]")), - FormatTreeNode::new(format!("partition by: [{partition_by}]")), - FormatTreeNode::new(format!("order by: [{order_by}]")), - FormatTreeNode::new(format!("frame: [{frame}]")), - ]; - - if let Some(limit) = plan.limit { - children.push(FormatTreeNode::new(format!("limit: [{limit}]"))) - } - - append_profile_info(&mut children, profs, plan.plan_id); - - children.push(to_format_tree(&plan.input, metadata, profs, context)?); - - Ok(FormatTreeNode::with_children( - "Window".to_string(), - children, - )) -} - -fn sort_to_format_tree( - plan: &Sort, - metadata: &Metadata, - prof_span_set: &HashMap, - context: &mut FormatContext, -) -> Result> { - let sort_keys = plan - .order_by - .iter() - .map(|sort_key| { - Ok(format!( - "{} {} {}", - sort_key.display_name, - if sort_key.asc { "ASC" } else { "DESC" }, - if sort_key.nulls_first { - "NULLS FIRST" - } else { - "NULLS LAST" - } - )) - }) - .collect::>>()? - .join(", "); - - let mut children = vec![ - FormatTreeNode::new(format!( - "output columns: [{}]", - format_output_columns(plan.output_schema()?, metadata, true) - )), - FormatTreeNode::new(format!("sort keys: [{sort_keys}]")), - ]; - - if let Some(info) = &plan.stat_info { - let items = plan_stats_info_to_format_tree(info); - children.extend(items); - } - - append_profile_info(&mut children, prof_span_set, plan.plan_id); - - children.push(to_format_tree( - &plan.input, - metadata, - prof_span_set, - context, - )?); - - Ok(FormatTreeNode::with_children("Sort".to_string(), children)) -} - -fn window_partition_to_format_tree( - plan: &WindowPartition, - metadata: &Metadata, - prof_span_set: &HashMap, - context: &mut FormatContext, -) -> Result> { - let partition_by = plan - .partition_by - .iter() - .map(|&index| { - let name = metadata.column(index).name(); - Ok(name) - }) - .collect::>>()? - .join(", "); - - let mut children = vec![ - FormatTreeNode::new(format!( - "output columns: [{}]", - format_output_columns(plan.output_schema()?, metadata, true) - )), - FormatTreeNode::new(format!("hash keys: [{partition_by}]")), - ]; - - if let Some(top_n) = &plan.top_n { - children.push(FormatTreeNode::new(format!("top: {}", top_n.top))); - } - - if let Some(info) = &plan.stat_info { - let items = plan_stats_info_to_format_tree(info); - children.extend(items); - } - - append_profile_info(&mut children, prof_span_set, plan.plan_id); - - children.push(to_format_tree( - &plan.input, - metadata, - prof_span_set, - context, - )?); - - Ok(FormatTreeNode::with_children( - "WindowPartition".to_string(), - children, - )) -} - -fn limit_to_format_tree( - plan: &Limit, - metadata: &Metadata, - profs: &HashMap, - context: &mut FormatContext, -) -> Result> { - let mut children = vec![ - FormatTreeNode::new(format!( - "output columns: [{}]", - format_output_columns(plan.output_schema()?, metadata, true) - )), - FormatTreeNode::new(format!( - "limit: {}", - plan.limit - .map_or("NONE".to_string(), |limit| limit.to_string()) - )), - FormatTreeNode::new(format!("offset: {}", plan.offset)), - ]; - - if let Some(info) = &plan.stat_info { - let items = plan_stats_info_to_format_tree(info); - children.extend(items); - } - - append_profile_info(&mut children, profs, plan.plan_id); - - children.push(to_format_tree(&plan.input, metadata, profs, context)?); - - Ok(FormatTreeNode::with_children("Limit".to_string(), children)) -} - -fn row_fetch_to_format_tree( - plan: &RowFetch, - metadata: &Metadata, - profs: &HashMap, - context: &mut FormatContext, -) -> Result> { - let table_schema = plan.source.source_info.schema(); - let projected_schema = plan.cols_to_fetch.project_schema(&table_schema); - let fields_to_fetch = projected_schema.fields(); - - let mut children = vec![ - FormatTreeNode::new(format!( - "output columns: [{}]", - format_output_columns(plan.output_schema()?, metadata, true) - )), - FormatTreeNode::new(format!( - "columns to fetch: [{}]", - fields_to_fetch.iter().map(|f| f.name()).join(", ") - )), - ]; - - if let Some(info) = &plan.stat_info { - let items = plan_stats_info_to_format_tree(info); - children.extend(items); - } - - append_profile_info(&mut children, profs, plan.plan_id); - - children.push(to_format_tree(&plan.input, metadata, profs, context)?); - - Ok(FormatTreeNode::with_children( - "RowFetch".to_string(), - children, - )) -} - -fn range_join_to_format_tree( - plan: &RangeJoin, - metadata: &Metadata, - profs: &HashMap, - context: &mut FormatContext, -) -> Result> { - let range_join_conditions = plan - .conditions - .iter() - .map(|condition| { - let left = condition - .left_expr - .as_expr(&BUILTIN_FUNCTIONS) - .sql_display(); - let right = condition - .right_expr - .as_expr(&BUILTIN_FUNCTIONS) - .sql_display(); - format!("{left} {:?} {right}", condition.operator) - }) - .collect::>() - .join(", "); - let other_conditions = plan - .other_conditions - .iter() - .map(|filter| filter.as_expr(&BUILTIN_FUNCTIONS).sql_display()) - .collect::>() - .join(", "); - - let mut left_child = to_format_tree(&plan.left, metadata, profs, context)?; - let mut right_child = to_format_tree(&plan.right, metadata, profs, context)?; - - left_child.payload = format!("{}(Left)", left_child.payload); - right_child.payload = format!("{}(Right)", right_child.payload); - - let mut children = vec![ - FormatTreeNode::new(format!( - "output columns: [{}]", - format_output_columns(plan.output_schema()?, metadata, true) - )), - FormatTreeNode::new(format!("join type: {}", plan.join_type)), - FormatTreeNode::new(format!("range join conditions: [{range_join_conditions}]")), - FormatTreeNode::new(format!("other conditions: [{other_conditions}]")), - ]; - - if let Some(info) = &plan.stat_info { - let items = plan_stats_info_to_format_tree(info); - children.extend(items); - } - - append_profile_info(&mut children, profs, plan.plan_id); - - children.push(left_child); - children.push(right_child); - - Ok(FormatTreeNode::with_children( - match plan.range_join_type { - RangeJoinType::IEJoin => "IEJoin".to_string(), - RangeJoinType::Merge => "MergeJoin".to_string(), - }, - children, - )) -} - -fn hash_join_to_format_tree( - plan: &HashJoin, - metadata: &Metadata, - profs: &HashMap, - context: &mut FormatContext, -) -> Result> { - for rf in plan.runtime_filter.filters.iter() { - context - .scan_id_to_runtime_filters - .entry(rf.scan_id) - .or_default() - .push(rf.clone()); - } - let build_keys = plan - .build_keys - .iter() - .map(|scalar| scalar.as_expr(&BUILTIN_FUNCTIONS).sql_display()) - .collect::>() - .join(", "); - let probe_keys = plan - .probe_keys - .iter() - .map(|scalar| scalar.as_expr(&BUILTIN_FUNCTIONS).sql_display()) - .collect::>() - .join(", "); - let is_null_equal = plan.is_null_equal.iter().map(|b| format!("{b}")).join(", "); - let filters = plan - .non_equi_conditions - .iter() - .map(|filter| filter.as_expr(&BUILTIN_FUNCTIONS).sql_display()) - .collect::>() - .join(", "); - - let mut build_child = to_format_tree(&plan.build, metadata, profs, context)?; - let mut probe_child = to_format_tree(&plan.probe, metadata, profs, context)?; - - build_child.payload = format!("{}(Build)", build_child.payload); - probe_child.payload = format!("{}(Probe)", probe_child.payload); - - let mut build_runtime_filters = vec![]; - for rf in plan.runtime_filter.filters.iter() { - let mut s = format!( - "filter id:{}, build key:{}, probe key:{}, filter type:", - rf.id, - rf.build_key.as_expr(&BUILTIN_FUNCTIONS).sql_display(), - rf.probe_key.as_expr(&BUILTIN_FUNCTIONS).sql_display(), - ); - if rf.enable_bloom_runtime_filter { - s += "bloom,"; - } - if rf.enable_inlist_runtime_filter { - s += "inlist,"; - } - if rf.enable_min_max_runtime_filter { - s += "min_max,"; - } - s = s.trim_end_matches(',').to_string(); - build_runtime_filters.push(FormatTreeNode::new(s)); - } - - let mut children = vec![ - FormatTreeNode::new(format!( - "output columns: [{}]", - format_output_columns(plan.output_schema()?, metadata, true) - )), - FormatTreeNode::new(format!("join type: {}", plan.join_type)), - FormatTreeNode::new(format!("build keys: [{build_keys}]")), - FormatTreeNode::new(format!("probe keys: [{probe_keys}]")), - FormatTreeNode::new(format!("keys is null equal: [{is_null_equal}]")), - FormatTreeNode::new(format!("filters: [{filters}]")), - ]; - - if !build_runtime_filters.is_empty() { - if plan.broadcast_id.is_some() { - children.push(FormatTreeNode::with_children( - format!("build join filters(distributed):"), - build_runtime_filters, - )); - } else { - children.push(FormatTreeNode::with_children( - format!("build join filters:"), - build_runtime_filters, - )); - } - } - - if let Some((cache_index, column_map)) = &plan.build_side_cache_info { - let mut column_indexes = column_map.keys().collect::>(); - column_indexes.sort(); - children.push(FormatTreeNode::new(format!("cache index: {}", cache_index))); - children.push(FormatTreeNode::new(format!( - "cache columns: {:?}", - column_indexes - ))); - } - - if let Some(info) = &plan.stat_info { - let items = plan_stats_info_to_format_tree(info); - children.extend(items); - } - - append_profile_info(&mut children, profs, plan.plan_id); - - children.push(build_child); - children.push(probe_child); - - Ok(FormatTreeNode::with_children( - "HashJoin".to_string(), - children, - )) -} - -fn exchange_to_format_tree( - plan: &Exchange, - metadata: &Metadata, - profs: &HashMap, - context: &mut FormatContext, -) -> Result> { - Ok(FormatTreeNode::with_children("Exchange".to_string(), vec![ - FormatTreeNode::new(format!( - "output columns: [{}]", - format_output_columns(plan.output_schema()?, metadata, true) - )), - FormatTreeNode::new(format!("exchange type: {}", match plan.kind { - FragmentKind::Init => "Init-Partition".to_string(), - FragmentKind::Normal => format!( - "Hash({})", - plan.keys - .iter() - .map(|key| { key.as_expr(&BUILTIN_FUNCTIONS).sql_display() }) - .collect::>() - .join(", ") - ), - FragmentKind::Expansive => "Broadcast".to_string(), - FragmentKind::Merge => "Merge".to_string(), - })), - to_format_tree(&plan.input, metadata, profs, context)?, - ])) -} - -fn union_all_to_format_tree( - plan: &UnionAll, - metadata: &Metadata, - profs: &HashMap, - context: &mut FormatContext, -) -> Result> { - let mut children = vec![FormatTreeNode::new(format!( - "output columns: [{}]", - format_output_columns(plan.output_schema()?, metadata, true) - ))]; - - if let Some(info) = &plan.stat_info { - let items = plan_stats_info_to_format_tree(info); - children.extend(items); - } - - append_profile_info(&mut children, profs, plan.plan_id); - - children.extend(vec![ - to_format_tree(&plan.left, metadata, profs, context)?, - to_format_tree(&plan.right, metadata, profs, context)?, - ]); - - let root = if !plan.cte_scan_names.is_empty() { - "UnionAll(recursive cte)".to_string() - } else { - "UnionAll".to_string() - }; - - Ok(FormatTreeNode::with_children(root, children)) -} - -fn part_stats_info_to_format_tree(info: &PartStatistics) -> Vec> { +pub fn part_stats_info_to_format_tree(info: &PartStatistics) -> Vec> { let read_size = format_byte_size(info.read_bytes); let mut items = vec![ FormatTreeNode::new(format!("read rows: {}", info.read_rows)), @@ -1764,155 +206,14 @@ fn part_stats_info_to_format_tree(info: &PartStatistics) -> Vec Vec> { +pub fn plan_stats_info_to_format_tree(info: &PlanStatsInfo) -> Vec> { vec![FormatTreeNode::new(format!( "estimated rows: {0:.2}", info.estimated_rows ))] } -fn exchange_source_to_format_tree( - plan: &ExchangeSource, - metadata: &Metadata, -) -> Result> { - let mut children = vec![FormatTreeNode::new(format!( - "output columns: [{}]", - format_output_columns(plan.output_schema()?, metadata, true) - ))]; - - children.push(FormatTreeNode::new(format!( - "source fragment: [{}]", - plan.source_fragment_id - ))); - - Ok(FormatTreeNode::with_children( - "ExchangeSource".to_string(), - children, - )) -} - -fn exchange_sink_to_format_tree( - plan: &ExchangeSink, - metadata: &Metadata, - profs: &HashMap, - context: &mut FormatContext, -) -> Result> { - let mut children = vec![FormatTreeNode::new(format!( - "output columns: [{}]", - format_output_columns(plan.output_schema()?, metadata, true) - ))]; - - children.push(FormatTreeNode::new(format!( - "destination fragment: [{}]", - plan.destination_fragment_id - ))); - - children.push(to_format_tree(&plan.input, metadata, profs, context)?); - - Ok(FormatTreeNode::with_children( - "ExchangeSink".to_string(), - children, - )) -} - -fn distributed_insert_to_format_tree( - plan: &DistributedInsertSelect, - metadata: &Metadata, - profs: &HashMap, - context: &mut FormatContext, -) -> Result> { - let children = vec![to_format_tree(&plan.input, metadata, profs, context)?]; - - Ok(FormatTreeNode::with_children( - "DistributedInsertSelect".to_string(), - children, - )) -} - -fn commit_sink_to_format_tree( - plan: &CommitSink, - metadata: &Metadata, - profs: &HashMap, - context: &mut FormatContext, -) -> Result> { - let children = vec![to_format_tree(&plan.input, metadata, profs, context)?]; - Ok(FormatTreeNode::with_children( - "CommitSink".to_string(), - children, - )) -} - -fn project_set_to_format_tree( - plan: &ProjectSet, - metadata: &Metadata, - profs: &HashMap, - context: &mut FormatContext, -) -> Result> { - let mut children = vec![FormatTreeNode::new(format!( - "output columns: [{}]", - format_output_columns(plan.output_schema()?, metadata, true) - ))]; - - if let Some(info) = &plan.stat_info { - let items = plan_stats_info_to_format_tree(info); - children.extend(items); - } - - append_profile_info(&mut children, profs, plan.plan_id); - - children.extend(vec![FormatTreeNode::new(format!( - "set returning functions: {}", - plan.srf_exprs - .iter() - .map(|(expr, _)| expr.clone().as_expr(&BUILTIN_FUNCTIONS).sql_display()) - .collect::>() - .join(", ") - ))]); - - children.extend(vec![to_format_tree(&plan.input, metadata, profs, context)?]); - - Ok(FormatTreeNode::with_children( - "ProjectSet".to_string(), - children, - )) -} - -fn udf_to_format_tree( - plan: &Udf, - metadata: &Metadata, - profs: &HashMap, - context: &mut FormatContext, -) -> Result> { - let mut children = vec![FormatTreeNode::new(format!( - "output columns: [{}]", - format_output_columns(plan.output_schema()?, metadata, true) - ))]; - - if let Some(info) = &plan.stat_info { - let items = plan_stats_info_to_format_tree(info); - children.extend(items); - } - - append_profile_info(&mut children, profs, plan.plan_id); - - children.extend(vec![FormatTreeNode::new(format!( - "udf functions: {}", - plan.udf_funcs - .iter() - .map(|func| { - let arg_exprs = func.arg_exprs.join(", "); - format!("{}({})", func.func_name, arg_exprs) - }) - .collect::>() - .join(", ") - ))]); - - children.extend(vec![to_format_tree(&plan.input, metadata, profs, context)?]); - - Ok(FormatTreeNode::with_children("Udf".to_string(), children)) -} - -fn format_output_columns( +pub fn format_output_columns( output_schema: DataSchemaRef, metadata: &Metadata, format_table: bool, diff --git a/src/query/sql/src/executor/mod.rs b/src/query/sql/src/executor/mod.rs index 5b74fc81d1794..a84af92bfd606 100644 --- a/src/query/sql/src/executor/mod.rs +++ b/src/query/sql/src/executor/mod.rs @@ -22,14 +22,12 @@ mod util; pub mod table_read_plan; -pub use format::format_partial_tree; +pub use physical_plan::DeriveHandle; pub use physical_plan::IPhysicalPlan; pub use physical_plan::PhysicalPlan; -pub use physical_plan::PhysicalPlanDeriveHandle; pub use physical_plan::PhysicalPlanMeta; pub use physical_plan_builder::MutationBuildInfo; pub use physical_plan_builder::PhysicalPlanBuilder; -pub use physical_plan_visitor::PhysicalPlanReplacer; pub use physical_plans::build_broadcast_plans; pub use physical_plans::PhysicalRuntimeFilter; pub use physical_plans::PhysicalRuntimeFilters; diff --git a/src/query/sql/src/executor/physical_plan.rs b/src/query/sql/src/executor/physical_plan.rs index 9a5ea7edcdfb6..2de83edceef74 100644 --- a/src/query/sql/src/executor/physical_plan.rs +++ b/src/query/sql/src/executor/physical_plan.rs @@ -16,6 +16,8 @@ use std::any::Any; use std::collections::HashMap; use std::fmt::Debug; +use databend_common_ast::ast::FormatTreeNode; +use databend_common_base::runtime::profile::get_statistics_desc; use databend_common_catalog::plan::DataSourceInfo; use databend_common_catalog::plan::DataSourcePlan; use databend_common_catalog::plan::PartStatistics; @@ -23,6 +25,7 @@ use databend_common_catalog::plan::PartitionsShuffleKind; use databend_common_exception::Result; use databend_common_expression::DataSchemaRef; use databend_common_functions::BUILTIN_FUNCTIONS; +use databend_common_pipeline_core::PlanProfile; use educe::Educe; use enum_as_inner::EnumAsInner; use itertools::Itertools; @@ -35,6 +38,7 @@ use super::physical_plans::MutationManipulate; use super::physical_plans::MutationOrganize; use super::physical_plans::MutationSource; use super::physical_plans::MutationSplit; +use crate::executor::format::FormatContext; use crate::executor::physical_plans::AggregateExpand; use crate::executor::physical_plans::AggregateFinal; use crate::executor::physical_plans::AggregatePartial; @@ -80,6 +84,7 @@ use crate::executor::physical_plans::Udf; use crate::executor::physical_plans::UnionAll; use crate::executor::physical_plans::Window; use crate::executor::physical_plans::WindowPartition; +use crate::Metadata; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct PhysicalPlanMeta { @@ -89,11 +94,14 @@ pub struct PhysicalPlanMeta { impl PhysicalPlanMeta { pub fn new(name: impl Into) -> PhysicalPlanMeta { - PhysicalPlanMeta { plan_id: 0, name } + PhysicalPlanMeta { + plan_id: 0, + name: name.into(), + } } } -pub trait PhysicalPlanDeriveHandle { +pub trait DeriveHandle { fn derive( &mut self, v: &Box, @@ -108,7 +116,6 @@ pub trait IPhysicalPlan: Debug { fn get_meta_mut(&mut self) -> &mut PhysicalPlanMeta; // For methods with default implementations, the default implementation is usually sufficient. - fn get_id(&self) -> u32 { self.get_meta().plan_id } @@ -145,6 +152,14 @@ pub trait IPhysicalPlan: Debug { Box::new(std::iter::empty()) } + fn to_format_node( + &self, + _ctx: &mut FormatContext<'_>, + children: Vec>, + ) -> Result> { + Ok(FormatTreeNode::with_children(self.get_name(), children)) + } + /// Used to find data source info in a non-aggregation and single-table query plan. fn try_find_single_data_source(&self) -> Option<&DataSourcePlan> { None @@ -194,8 +209,7 @@ pub trait IPhysicalPlan: Debug { Ok(HashMap::new()) } - fn derive_with(&self, handle: &mut Box) - -> Box; + fn derive(&self, children: Vec>) -> Box; } pub trait PhysicalPlanExt { @@ -210,8 +224,72 @@ where T: 'static + IPhysicalPlan + Clone fn clone_box(&self) -> Box { Box::new(self.clone()) } + + fn down_cast(&self) -> Option<&T> { + todo!() + } } +pub trait PhysicalPlanDynExt { + fn format( + &self, + metadata: &Metadata, + profs: HashMap, + ) -> Result> { + let mut context = FormatContext { + metadata, + scan_id_to_runtime_filters: HashMap::new(), + }; + + self.to_format_tree(&profs, &mut context) + } + + fn to_format_tree( + &self, + profs: &HashMap, + ctx: &mut FormatContext<'_>, + ) -> Result>; +} + +impl PhysicalPlanDynExt for Box { + fn to_format_tree( + &self, + profs: &HashMap, + ctx: &mut FormatContext<'_>, + ) -> Result> { + let mut children = Vec::with_capacity(4); + for child in self.children() { + children.push(child.to_format_tree(profs, ctx)?); + } + + let mut format_tree_node = self.to_format_node(ctx, children)?; + + if let Some(prof) = profs.get(&self.get_id()) { + let mut children = Vec::with_capacity(format_tree_node.children.len() + 10); + for (_, desc) in get_statistics_desc().iter() { + if desc.display_name != "output rows" { + continue; + } + if prof.statistics[desc.index] != 0 { + children.push(FormatTreeNode::new(format!( + "{}: {}", + desc.display_name.to_lowercase(), + desc.human_format(prof.statistics[desc.index]) + ))); + } + break; + } + + children.append(&mut format_tree_node.children); + format_tree_node.children = children; + } + + Ok(format_tree_node) + } +} + +unsafe impl Send for dyn IPhysicalPlan {} + impl Clone for Box { fn clone(&self) -> Self { self.clone_box() diff --git a/src/query/sql/src/executor/physical_plans/physical_add_stream_column.rs b/src/query/sql/src/executor/physical_plans/physical_add_stream_column.rs index 2476871e99ccd..1fb5617730eed 100644 --- a/src/query/sql/src/executor/physical_plans/physical_add_stream_column.rs +++ b/src/query/sql/src/executor/physical_plans/physical_add_stream_column.rs @@ -14,6 +14,7 @@ use std::collections::HashMap; +use databend_common_ast::ast::FormatTreeNode; use databend_common_catalog::plan::StreamColumn; use databend_common_catalog::plan::StreamColumnType; use databend_common_exception::Result; @@ -23,8 +24,8 @@ use databend_common_expression::ORIGIN_BLOCK_ID_COL_NAME; use databend_common_expression::ORIGIN_BLOCK_ROW_NUM_COL_NAME; use databend_common_expression::ORIGIN_VERSION_COL_NAME; -use crate::executor::physical_plan::DeriveExt; -use crate::executor::physical_plan::PhysicalPlanDeriveHandle; +use crate::executor::format::FormatContext; +use crate::executor::physical_plan::DeriveHandle; use crate::executor::IPhysicalPlan; use crate::executor::PhysicalPlan; use crate::executor::PhysicalPlanMeta; @@ -68,21 +69,21 @@ impl IPhysicalPlan for AddStreamColumn { Box::new(std::iter::once(&mut self.input)) } - fn derive_with( + fn to_format_node( &self, - handle: &mut Box, - ) -> Box { - let derive_input = self.input.derive_with(handle); - - match handle.derive(self, vec![derive_input]) { - Ok(v) => v, - Err(children) => { - let mut new_add_stream_column = self.clone(); - assert_eq!(children.len(), 1); - new_add_stream_column.input = children[0]; - Box::new(new_add_stream_column) - } - } + _: &mut FormatContext<'_>, + mut children: Vec>, + ) -> Result> { + // ignore self + assert_eq!(children.len(), 1); + Ok(children.pop().unwrap()) + } + + fn derive(&self, mut children: Vec>) -> Box { + let mut new_physical_plan = self.clone(); + assert_eq!(children.len(), 1); + new_physical_plan.input = children.pop().unwrap(); + Box::new(new_physical_plan) } } diff --git a/src/query/sql/src/executor/physical_plans/physical_aggregate_expand.rs b/src/query/sql/src/executor/physical_plans/physical_aggregate_expand.rs index 80dd78b526c3f..a0713263c35d5 100644 --- a/src/query/sql/src/executor/physical_plans/physical_aggregate_expand.rs +++ b/src/query/sql/src/executor/physical_plans/physical_aggregate_expand.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use databend_common_ast::ast::FormatTreeNode; use databend_common_catalog::plan::DataSourcePlan; use databend_common_exception::Result; use databend_common_expression::types::DataType; @@ -21,7 +22,10 @@ use databend_common_expression::DataSchemaRef; use databend_common_expression::DataSchemaRefExt; use crate::executor::explain::PlanStatsInfo; -use crate::executor::physical_plan::PhysicalPlanDeriveHandle; +use crate::executor::format::format_output_columns; +use crate::executor::format::plan_stats_info_to_format_tree; +use crate::executor::format::FormatContext; +use crate::executor::physical_plan::DeriveHandle; use crate::executor::IPhysicalPlan; use crate::executor::PhysicalPlan; use crate::executor::PhysicalPlanMeta; @@ -87,6 +91,45 @@ impl IPhysicalPlan for AggregateExpand { Box::new(std::iter::once(&mut self.input)) } + fn to_format_node( + &self, + ctx: &mut FormatContext<'_>, + children: Vec>, + ) -> Result> { + let sets = self + .grouping_sets + .sets + .iter() + .map(|set| { + set.iter() + .map(|&index| ctx.metadata.column(index).name()) + .collect::>() + .join(", ") + }) + .map(|s| format!("({})", s)) + .collect::>() + .join(", "); + + let mut node_children = vec![ + FormatTreeNode::new(format!( + "output columns: [{}]", + format_output_columns(self.output_schema()?, &ctx.metadata, true) + )), + FormatTreeNode::new(format!("grouping sets: [{sets}]")), + ]; + + if let Some(info) = &self.stat_info { + let items = plan_stats_info_to_format_tree(info); + node_children.extend(items); + } + + node_children.extend(children); + Ok(FormatTreeNode::with_children( + "AggregateExpand".to_string(), + node_children, + )) + } + fn get_desc(&self) -> Result { Ok(self .grouping_sets @@ -103,20 +146,10 @@ impl IPhysicalPlan for AggregateExpand { .join(", ")) } - fn derive_with( - &self, - handle: &mut Box, - ) -> Box { - let derive_input = self.input.derive_with(handle); - - match handle.derive(self, vec![derive_input]) { - Ok(v) => v, - Err(children) => { - let mut new_aggregate_expand = self.clone(); - assert_eq!(children.len(), 1); - new_aggregate_expand.input = children[0]; - Box::new(new_aggregate_expand) - } - } + fn derive(&self, mut children: Vec>) -> Box { + let mut new_physical_plan = self.clone(); + assert_eq!(children.len(), 1); + new_physical_plan.input = children.pop().unwrap(); + Box::new(new_physical_plan) } } diff --git a/src/query/sql/src/executor/physical_plans/physical_aggregate_final.rs b/src/query/sql/src/executor/physical_plans/physical_aggregate_final.rs index 8fa3ae507482a..7c7e7c539c229 100644 --- a/src/query/sql/src/executor/physical_plans/physical_aggregate_final.rs +++ b/src/query/sql/src/executor/physical_plans/physical_aggregate_final.rs @@ -15,6 +15,7 @@ use std::collections::HashMap; use std::sync::Arc; +use databend_common_ast::ast::FormatTreeNode; use databend_common_catalog::plan::DataSourcePlan; use databend_common_exception::ErrorCode; use databend_common_exception::Result; @@ -26,7 +27,11 @@ use itertools::Itertools; use super::SortDesc; use crate::executor::explain::PlanStatsInfo; -use crate::executor::physical_plan::PhysicalPlanDeriveHandle; +use crate::executor::format::format_output_columns; +use crate::executor::format::plan_stats_info_to_format_tree; +use crate::executor::format::pretty_display_agg_desc; +use crate::executor::format::FormatContext; +use crate::executor::physical_plan::DeriveHandle; use crate::executor::physical_plans::AggregateExpand; use crate::executor::physical_plans::AggregateFunctionDesc; use crate::executor::physical_plans::AggregateFunctionSignature; @@ -93,6 +98,49 @@ impl IPhysicalPlan for AggregateFinal { Box::new(std::iter::once(&mut self.input)) } + fn to_format_node( + &self, + ctx: &mut FormatContext<'_>, + children: Vec>, + ) -> Result> { + let group_by = self + .group_by + .iter() + .map(|&index| { + let name = ctx.metadata.column(index).name(); + Ok(name) + }) + .collect::>>()? + .join(", "); + + let agg_funcs = self + .agg_funcs + .iter() + .map(|agg| pretty_display_agg_desc(agg, &ctx.metadata)) + .collect::>() + .join(", "); + + let mut node_children = vec![ + FormatTreeNode::new(format!( + "output columns: [{}]", + format_output_columns(self.output_schema()?, &ctx.metadata, true) + )), + FormatTreeNode::new(format!("group by: [{group_by}]")), + FormatTreeNode::new(format!("aggregate functions: [{agg_funcs}]")), + ]; + + if let Some(info) = &self.stat_info { + let items = plan_stats_info_to_format_tree(info); + node_children.extend(items); + } + + node_children.extend(children); + Ok(FormatTreeNode::with_children( + "AggregateFinal".to_string(), + node_children, + )) + } + fn get_desc(&self) -> Result { Ok(self.agg_funcs.iter().map(|x| x.display.clone()).join(", ")) } @@ -113,21 +161,11 @@ impl IPhysicalPlan for AggregateFinal { Ok(labels) } - fn derive_with( - &self, - handle: &mut Box, - ) -> Box { - let derive_input = self.input.derive_with(handle); - - match handle.derive(self, vec![derive_input]) { - Ok(v) => v, - Err(children) => { - let mut new_aggregate_final = self.clone(); - assert_eq!(children.len(), 1); - new_aggregate_final.input = children[0]; - Box::new(new_aggregate_final) - } - } + fn derive(&self, mut children: Vec>) -> Box { + let mut new_physical_plan = self.clone(); + assert_eq!(children.len(), 1); + new_physical_plan.input = children.pop().unwrap(); + Box::new(new_physical_plan) } } @@ -173,7 +211,7 @@ impl PhysicalPlanBuilder { let input_schema = input.output_schema()?; let group_items = agg.group_items.iter().map(|v| v.index).collect::>(); - let result = match &agg.mode { + let result: Box = match &agg.mode { AggregateMode::Partial => { let group_by_display = agg .group_items @@ -327,16 +365,16 @@ impl PhysicalPlanBuilder { if group_by_shuffle_mode == "before_merge" => { let aggregate_partial = if let Some(grouping_sets) = agg.grouping_sets { - let expand = AggregateExpand { + let expand = Box::new(AggregateExpand { input, grouping_sets, group_bys: group_items.clone(), stat_info: Some(stat_info.clone()), meta: PhysicalPlanMeta::new("AggregateExpand"), - }; + }); - AggregatePartial { - input: Box::new(PhysicalPlan::AggregateExpand(expand)), + Box::new(AggregatePartial { + input: expand, agg_funcs, enable_experimental_aggregate_hashtable, group_by_display, @@ -344,9 +382,9 @@ impl PhysicalPlanBuilder { stat_info: Some(stat_info), rank_limit: None, meta: PhysicalPlanMeta::new("AggregatePartial"), - } + }) } else { - AggregatePartial { + Box::new(AggregatePartial { input, agg_funcs, rank_limit, @@ -355,7 +393,7 @@ impl PhysicalPlanBuilder { group_by: group_items, stat_info: Some(stat_info), meta: PhysicalPlanMeta::new("AggregatePartial"), - } + }) }; let keys = { @@ -372,20 +410,20 @@ impl PhysicalPlanBuilder { .collect() }; - PhysicalPlan::Exchange(Exchange { + Box::new(Exchange { kind, keys, ignore_exchange: false, allow_adjust_parallelism: true, meta: PhysicalPlanMeta::new("Exchange"), - input: Box::new(PhysicalPlan::AggregatePartial(aggregate_partial)), + input: aggregate_partial, }) } _ => { if let Some(grouping_sets) = agg.grouping_sets { let expand = AggregateExpand { + input, grouping_sets, - input: Box::new(input), group_bys: group_items.clone(), stat_info: Some(stat_info.clone()), meta: PhysicalPlanMeta::new("AggregateExpand"), @@ -402,12 +440,12 @@ impl PhysicalPlanBuilder { meta: PhysicalPlanMeta::new("AggregatePartial"), }) } else { - PhysicalPlan::AggregatePartial(AggregatePartial { + Box::new(AggregatePartial { + input, agg_funcs, enable_experimental_aggregate_hashtable, group_by_display, group_by: group_items, - input: Box::new(input), stat_info: Some(stat_info), rank_limit, meta: PhysicalPlanMeta::new("AggregatePartial"), @@ -430,7 +468,7 @@ impl PhysicalPlanBuilder { _ => { return Err(ErrorCode::Internal(format!( "invalid input physical plan: {}", - input.name(), + input.get_name(), ))); } }; @@ -556,15 +594,16 @@ impl PhysicalPlanBuilder { match input { PhysicalPlan::AggregatePartial(ref partial) => { + let group_by_display = partial.group_by_display.clone(); let before_group_by_schema = partial.input.output_schema()?; - PhysicalPlan::AggregateFinal(AggregateFinal { + Box::new(AggregateFinal { + input, agg_funcs, + group_by_display, before_group_by_schema, group_by: group_items, - input: Box::new(input), stat_info: Some(stat_info), - group_by_display: partial.group_by_display.clone(), meta: PhysicalPlanMeta::new("AggregateFinal"), }) } @@ -576,10 +615,10 @@ impl PhysicalPlanBuilder { let before_group_by_schema = partial.input.output_schema()?; Box::new(AggregateFinal { + input, agg_funcs, before_group_by_schema, group_by: group_items, - input: Box::new(input), stat_info: Some(stat_info), group_by_display: partial.group_by_display.clone(), meta: PhysicalPlanMeta::new("AggregateFinal"), @@ -589,7 +628,7 @@ impl PhysicalPlanBuilder { _ => { return Err(ErrorCode::Internal(format!( "invalid input physical plan: {}", - input.name(), + input.get_name(), ))); } } diff --git a/src/query/sql/src/executor/physical_plans/physical_aggregate_partial.rs b/src/query/sql/src/executor/physical_plans/physical_aggregate_partial.rs index 24d0e103ae439..636b113bcb0e6 100644 --- a/src/query/sql/src/executor/physical_plans/physical_aggregate_partial.rs +++ b/src/query/sql/src/executor/physical_plans/physical_aggregate_partial.rs @@ -14,6 +14,7 @@ use std::collections::HashMap; +use databend_common_ast::ast::FormatTreeNode; use databend_common_catalog::plan::DataSourcePlan; use databend_common_exception::Result; use databend_common_expression::types::DataType; @@ -26,7 +27,10 @@ use itertools::Itertools; use super::SortDesc; use crate::executor::explain::PlanStatsInfo; -use crate::executor::physical_plan::PhysicalPlanDeriveHandle; +use crate::executor::format::plan_stats_info_to_format_tree; +use crate::executor::format::pretty_display_agg_desc; +use crate::executor::format::FormatContext; +use crate::executor::physical_plan::DeriveHandle; use crate::executor::physical_plans::common::AggregateFunctionDesc; use crate::executor::IPhysicalPlan; use crate::executor::PhysicalPlan; @@ -89,6 +93,45 @@ impl IPhysicalPlan for AggregatePartial { Box::new(std::iter::once(&mut self.input)) } + fn to_format_node( + &self, + ctx: &mut FormatContext<'_>, + children: Vec>, + ) -> Result> { + let group_by = self + .group_by + .iter() + .map(|&index| ctx.metadata.column(index).name()) + .join(", "); + + let agg_funcs = self + .agg_funcs + .iter() + .map(|agg| pretty_display_agg_desc(agg, &ctx.metadata)) + .collect::>() + .join(", "); + + let mut node_children = vec![ + FormatTreeNode::new(format!("group by: [{group_by}]")), + FormatTreeNode::new(format!("aggregate functions: [{agg_funcs}]")), + ]; + + if let Some(info) = &self.stat_info { + node_children.extend(plan_stats_info_to_format_tree(info)); + } + + if let Some((_, r)) = &self.rank_limit { + node_children.push(FormatTreeNode::new(format!("rank limit: {r}"))); + } + + node_children.extend(children); + + Ok(FormatTreeNode::with_children( + "AggregatePartial".to_string(), + node_children, + )) + } + fn get_desc(&self) -> Result { Ok(self.agg_funcs.iter().map(|x| x.display.clone()).join(", ")) } @@ -97,7 +140,7 @@ impl IPhysicalPlan for AggregatePartial { let mut labels = HashMap::with_capacity(2); if !self.group_by_display.is_empty() { - labels.insert(String::from("Grouping keys"), v.group_by_display.clone()); + labels.insert(String::from("Grouping keys"), self.group_by_display.clone()); } if !self.agg_funcs.is_empty() { @@ -110,20 +153,10 @@ impl IPhysicalPlan for AggregatePartial { Ok(labels) } - fn derive_with( - &self, - handle: &mut Box, - ) -> Box { - let derive_input = self.input.derive_with(handle); - - match handle.derive(self, vec![derive_input]) { - Ok(v) => v, - Err(children) => { - let mut new_aggregate_partial = self.clone(); - assert_eq!(children.len(), 1); - new_aggregate_partial.input = children[0]; - Box::new(new_aggregate_partial) - } - } + fn derive(&self, mut children: Vec>) -> Box { + let mut new_physical_plan = self.clone(); + assert_eq!(children.len(), 1); + new_physical_plan.input = children.pop().unwrap(); + Box::new(new_physical_plan) } } diff --git a/src/query/sql/src/executor/physical_plans/physical_async_func.rs b/src/query/sql/src/executor/physical_plans/physical_async_func.rs index f481b69806357..a52d84bbeb02e 100644 --- a/src/query/sql/src/executor/physical_plans/physical_async_func.rs +++ b/src/query/sql/src/executor/physical_plans/physical_async_func.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use databend_common_ast::ast::FormatTreeNode; use databend_common_catalog::plan::DataSourcePlan; use databend_common_exception::ErrorCode; use databend_common_exception::Result; @@ -22,7 +23,10 @@ use databend_common_expression::DataSchemaRefExt; use itertools::Itertools; use crate::executor::explain::PlanStatsInfo; -use crate::executor::physical_plan::PhysicalPlanDeriveHandle; +use crate::executor::format::format_output_columns; +use crate::executor::format::plan_stats_info_to_format_tree; +use crate::executor::format::FormatContext; +use crate::executor::physical_plan::DeriveHandle; use crate::executor::IPhysicalPlan; use crate::executor::PhysicalPlan; use crate::executor::PhysicalPlanBuilder; @@ -73,6 +77,28 @@ impl IPhysicalPlan for AsyncFunction { Box::new(std::iter::once(&mut self.input)) } + fn to_format_node( + &self, + ctx: &mut FormatContext<'_>, + children: Vec>, + ) -> Result> { + let mut node_children = vec![FormatTreeNode::new(format!( + "output columns: [{}]", + format_output_columns(self.output_schema()?, &ctx.metadata, true) + ))]; + + if let Some(info) = &self.stat_info { + node_children.extend(plan_stats_info_to_format_tree(info)); + } + + node_children.extend(children); + + Ok(FormatTreeNode::with_children( + "AsyncFunction".to_string(), + node_children, + )) + } + #[recursive::recursive] fn try_find_single_data_source(&self) -> Option<&DataSourcePlan> { self.input.try_find_single_data_source() @@ -86,21 +112,11 @@ impl IPhysicalPlan for AsyncFunction { .join(", ")) } - fn derive_with( - &self, - handle: &mut Box, - ) -> Box { - let derive_input = self.input.derive_with(handle); - - match handle.derive(self, vec![derive_input]) { - Ok(v) => v, - Err(children) => { - let mut new_async_function = self.clone(); - assert_eq!(children.len(), 1); - new_async_function.input = children[0]; - Box::new(new_async_function) - } - } + fn derive(&self, mut children: Vec>) -> Box { + let mut new_physical_plan = self.clone(); + assert_eq!(children.len(), 1); + new_physical_plan.input = children.pop().unwrap(); + Box::new(new_physical_plan) } } @@ -182,8 +198,8 @@ impl PhysicalPlanBuilder { .collect::>>()?; Ok(Box::new(AsyncFunction { + input, meta: PhysicalPlanMeta::new("AsyncFunction"), - input: Box::new(input), async_func_descs, stat_info: Some(stat_info), })) diff --git a/src/query/sql/src/executor/physical_plans/physical_broadcast.rs b/src/query/sql/src/executor/physical_plans/physical_broadcast.rs index e472836d1dd8c..2cea4c451a21a 100644 --- a/src/query/sql/src/executor/physical_plans/physical_broadcast.rs +++ b/src/query/sql/src/executor/physical_plans/physical_broadcast.rs @@ -12,13 +12,15 @@ // See the License for the specific language governing permissions and // limitations under the License. +use databend_common_ast::ast::FormatTreeNode; use databend_common_catalog::table_context::TableContext; use databend_common_exception::Result; use databend_common_expression::DataSchemaRef; use super::Exchange; use super::FragmentKind; -use crate::executor::physical_plan::PhysicalPlanDeriveHandle; +use crate::executor::format::FormatContext; +use crate::executor::physical_plan::DeriveHandle; use crate::executor::IPhysicalPlan; use crate::executor::PhysicalPlan; use crate::executor::PhysicalPlanMeta; @@ -39,17 +41,9 @@ impl IPhysicalPlan for BroadcastSource { &mut self.meta } - fn derive_with( - &self, - handle: &mut Box, - ) -> Box { - match handle.derive(self, vec![]) { - Ok(v) => v, - Err(children) => { - assert!(children.is_empty()); - Box::new(self.clone()) - } - } + fn derive(&self, children: Vec>) -> Box { + assert!(children.is_empty()); + Box::new(self.clone()) } } @@ -84,21 +78,20 @@ impl IPhysicalPlan for BroadcastSink { Box::new(std::iter::once(&mut self.input)) } - fn derive_with( + fn to_format_node( &self, - handle: &mut Box, - ) -> Box { - let derive_input = self.input.derive_with(handle); - - match handle.derive(self, vec![derive_input]) { - Ok(v) => v, - Err(children) => { - let mut new_broadcast_sink = self.clone(); - assert_eq!(children.len(), 1); - new_broadcast_sink.input = children[0]; - Box::new(new_broadcast_sink) - } - } + _ctx: &mut FormatContext<'_>, + _: Vec>, + ) -> Result> { + // ignore children + Ok(FormatTreeNode::new("RuntimeFilterSink".to_string())) + } + + fn derive(&self, mut children: Vec>) -> Box { + let mut new_physical_plan = self.clone(); + assert_eq!(children.len(), 1); + new_physical_plan.input = children.pop().unwrap(); + Box::new(new_physical_plan) } } diff --git a/src/query/sql/src/executor/physical_plans/physical_cache_scan.rs b/src/query/sql/src/executor/physical_plans/physical_cache_scan.rs index 5d21e033e48ae..ac464d95c4984 100644 --- a/src/query/sql/src/executor/physical_plans/physical_cache_scan.rs +++ b/src/query/sql/src/executor/physical_plans/physical_cache_scan.rs @@ -12,11 +12,14 @@ // See the License for the specific language governing permissions and // limitations under the License. +use databend_common_ast::ast::FormatTreeNode; use databend_common_exception::Result; use databend_common_expression::DataSchemaRef; use databend_common_expression::DataSchemaRefExt; -use crate::executor::physical_plan::PhysicalPlanDeriveHandle; +use crate::executor::format::format_output_columns; +use crate::executor::format::FormatContext; +use crate::executor::physical_plan::DeriveHandle; use crate::executor::IPhysicalPlan; use crate::executor::PhysicalPlan; use crate::executor::PhysicalPlanBuilder; @@ -45,17 +48,38 @@ impl IPhysicalPlan for CacheScan { Ok(self.output_schema.clone()) } - fn derive_with( + fn to_format_node( &self, - handle: &mut Box, - ) -> Box { - match handle.derive(self, vec![]) { - Ok(v) => v, - Err(children) => { - assert!(children.is_empty()); - Box::new(self.clone()) + ctx: &mut FormatContext<'_>, + _: Vec>, + ) -> Result> { + let mut children = Vec::with_capacity(2); + children.push(FormatTreeNode::new(format!( + "output columns: [{}]", + format_output_columns(self.output_schema()?, &ctx.metadata, true) + ))); + + match &self.cache_source { + CacheSource::HashJoinBuild((cache_index, column_indexes)) => { + let mut column_indexes = column_indexes.clone(); + column_indexes.sort(); + children.push(FormatTreeNode::new(format!("cache index: {}", cache_index))); + children.push(FormatTreeNode::new(format!( + "column indexes: {:?}", + column_indexes + ))); } } + + Ok(FormatTreeNode::with_children( + "CacheScan".to_string(), + children, + )) + } + + fn derive(&self, children: Vec>) -> Box { + assert!(children.is_empty()); + Box::new(self.clone()) } } diff --git a/src/query/sql/src/executor/physical_plans/physical_column_mutation.rs b/src/query/sql/src/executor/physical_plans/physical_column_mutation.rs index 8c26c2a21ba3c..8648b6f471473 100644 --- a/src/query/sql/src/executor/physical_plans/physical_column_mutation.rs +++ b/src/query/sql/src/executor/physical_plans/physical_column_mutation.rs @@ -14,6 +14,7 @@ use std::collections::HashMap; +use databend_common_ast::ast::FormatTreeNode; use databend_common_catalog::plan::DataSourcePlan; use databend_common_exception::Result; use databend_common_expression::DataSchemaRef; @@ -21,8 +22,9 @@ use databend_common_expression::RemoteExpr; use databend_common_meta_app::schema::TableInfo; use databend_storages_common_table_meta::meta::TableMetaTimestamps; +use crate::executor::format::FormatContext; +use crate::executor::physical_plan::DeriveHandle; use crate::executor::physical_plan::PhysicalPlan; -use crate::executor::physical_plan::PhysicalPlanDeriveHandle; use crate::executor::physical_plans::MutationKind; use crate::executor::IPhysicalPlan; use crate::executor::PhysicalPlanMeta; @@ -65,20 +67,20 @@ impl IPhysicalPlan for ColumnMutation { Box::new(std::iter::once(&mut self.input)) } - fn derive_with( + fn to_format_node( &self, - handle: &mut Box, - ) -> Box { - let derive_input = self.input.derive_with(handle); + _: &mut FormatContext<'_>, + mut children: Vec>, + ) -> Result> { + // ignore self + assert_eq!(children.len(), 1); + Ok(children.pop().unwrap()) + } - match handle.derive(self, vec![derive_input]) { - Ok(v) => v, - Err(children) => { - let mut new_column_mutation = self.clone(); - assert_eq!(children.len(), 1); - new_column_mutation.input = children[0]; - Box::new(new_column_mutation) - } - } + fn derive(&self, mut children: Vec>) -> Box { + let mut new_physical_plan = self.clone(); + assert_eq!(children.len(), 1); + new_physical_plan.input = children.pop().unwrap(); + Box::new(new_physical_plan) } } diff --git a/src/query/sql/src/executor/physical_plans/physical_commit_sink.rs b/src/query/sql/src/executor/physical_plans/physical_commit_sink.rs index 4c1fed220a2d3..339ffffc0bf21 100644 --- a/src/query/sql/src/executor/physical_plans/physical_commit_sink.rs +++ b/src/query/sql/src/executor/physical_plans/physical_commit_sink.rs @@ -23,7 +23,7 @@ use databend_common_meta_app::schema::UpdateStreamMetaReq; use databend_storages_common_table_meta::meta::TableMetaTimestamps; use databend_storages_common_table_meta::meta::TableSnapshot; -use crate::executor::physical_plan::PhysicalPlanDeriveHandle; +use crate::executor::physical_plan::DeriveHandle; use crate::executor::physical_plans::common::MutationKind; use crate::executor::IPhysicalPlan; use crate::executor::PhysicalPlan; @@ -71,21 +71,11 @@ impl IPhysicalPlan for CommitSink { Box::new(std::iter::once(&mut self.input)) } - fn derive_with( - &self, - handle: &mut Box, - ) -> Box { - let derive_input = self.input.derive_with(handle); - - match handle.derive(self, vec![derive_input]) { - Ok(v) => v, - Err(children) => { - let mut new_commit_sink = self.clone(); - assert_eq!(children.len(), 1); - new_commit_sink.input = children[0]; - Box::new(new_commit_sink) - } - } + fn derive(&self, mut children: Vec>) -> Box { + let mut new_physical_plan = self.clone(); + assert_eq!(children.len(), 1); + new_physical_plan.input = children.pop().unwrap(); + Box::new(new_physical_plan) } } diff --git a/src/query/sql/src/executor/physical_plans/physical_compact_source.rs b/src/query/sql/src/executor/physical_plans/physical_compact_source.rs index f9f4666c308ca..daa11e1e3a2d3 100644 --- a/src/query/sql/src/executor/physical_plans/physical_compact_source.rs +++ b/src/query/sql/src/executor/physical_plans/physical_compact_source.rs @@ -25,7 +25,7 @@ use databend_common_expression::DataSchemaRef; use databend_common_meta_app::schema::TableInfo; use databend_storages_common_table_meta::meta::TableMetaTimestamps; -use crate::executor::physical_plan::PhysicalPlanDeriveHandle; +use crate::executor::physical_plan::DeriveHandle; use crate::executor::physical_plans::CommitSink; use crate::executor::physical_plans::CommitType; use crate::executor::physical_plans::Exchange; @@ -55,17 +55,9 @@ impl IPhysicalPlan for CompactSource { &mut self.meta } - fn derive_with( - &self, - handle: &mut Box, - ) -> Box { - match handle.derive(self, vec![]) { - Ok(v) => v, - Err(children) => { - assert!(children.is_empty()); - Box::new(self.clone()) - } - } + fn derive(&self, children: Vec>) -> Box { + assert!(children.is_empty()); + Box::new(self.clone()) } } @@ -123,7 +115,7 @@ impl PhysicalPlanBuilder { } root = Box::new(CommitSink { - input: Box::new(root), + input: root, table_info, snapshot: Some(snapshot), commit_type: CommitType::Mutation { diff --git a/src/query/sql/src/executor/physical_plans/physical_constant_table_scan.rs b/src/query/sql/src/executor/physical_plans/physical_constant_table_scan.rs index 41623b0c580cb..ccd6194661dec 100644 --- a/src/query/sql/src/executor/physical_plans/physical_constant_table_scan.rs +++ b/src/query/sql/src/executor/physical_plans/physical_constant_table_scan.rs @@ -12,11 +12,15 @@ // See the License for the specific language governing permissions and // limitations under the License. +use databend_common_ast::ast::FormatTreeNode; use databend_common_exception::Result; use databend_common_expression::Column; use databend_common_expression::DataSchemaRef; +use itertools::Itertools; -use crate::executor::physical_plan::PhysicalPlanDeriveHandle; +use crate::executor::format::format_output_columns; +use crate::executor::format::FormatContext; +use crate::executor::physical_plan::DeriveHandle; use crate::executor::IPhysicalPlan; use crate::executor::PhysicalPlan; use crate::executor::PhysicalPlanBuilder; @@ -46,17 +50,34 @@ impl IPhysicalPlan for ConstantTableScan { Ok(self.output_schema.clone()) } - fn derive_with( + fn to_format_node( &self, - handle: &mut Box, - ) -> Box { - match handle.derive(self, vec![]) { - Ok(v) => v, - Err(children) => { - assert!(children.is_empty()); - Box::new(self.clone()) - } + ctx: &mut FormatContext<'_>, + _: Vec>, + ) -> Result> { + if self.num_rows == 0 { + return Ok(FormatTreeNode::new(self.name().to_string())); + } + + let mut children = Vec::with_capacity(self.values.len() + 1); + children.push(FormatTreeNode::new(format!( + "output columns: [{}]", + format_output_columns(self.output_schema()?, &ctx.metadata, true) + ))); + for (i, value) in self.values.iter().enumerate() { + let column = value.iter().map(|val| format!("{val}")).join(", "); + children.push(FormatTreeNode::new(format!("column {}: [{}]", i, column))); } + + Ok(FormatTreeNode::with_children( + self.name().to_string(), + children, + )) + } + + fn derive(&self, children: Vec>) -> Box { + assert!(children.is_empty()); + Box::new(self.clone()) } } diff --git a/src/query/sql/src/executor/physical_plans/physical_copy_into_location.rs b/src/query/sql/src/executor/physical_plans/physical_copy_into_location.rs index c14d38fc6bce9..cb10eecdcb7d0 100644 --- a/src/query/sql/src/executor/physical_plans/physical_copy_into_location.rs +++ b/src/query/sql/src/executor/physical_plans/physical_copy_into_location.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use databend_common_ast::ast::FormatTreeNode; use databend_common_catalog::plan::DataSourcePlan; use databend_common_exception::Result; use databend_common_expression::types::DataType; @@ -22,7 +23,8 @@ use databend_common_expression::DataSchemaRefExt; use databend_common_expression::TableSchemaRef; use databend_storages_common_stage::CopyIntoLocationInfo; -use crate::executor::physical_plan::PhysicalPlanDeriveHandle; +use crate::executor::format::FormatContext; +use crate::executor::physical_plan::DeriveHandle; use crate::executor::IPhysicalPlan; use crate::executor::PhysicalPlan; use crate::executor::PhysicalPlanMeta; @@ -66,25 +68,26 @@ impl IPhysicalPlan for CopyIntoLocation { Box::new(std::iter::once(&mut self.input)) } + fn to_format_node( + &self, + _ctx: &mut FormatContext<'_>, + children: Vec>, + ) -> Result> { + Ok(FormatTreeNode::with_children( + "CopyIntoLocation".to_string(), + children, + )) + } + #[recursive::recursive] fn try_find_single_data_source(&self) -> Option<&DataSourcePlan> { self.input.try_find_single_data_source() } - fn derive_with( - &self, - handle: &mut Box, - ) -> Box { - let derive_input = self.input.derive_with(handle); - - match handle.derive(self, vec![derive_input]) { - Ok(v) => v, - Err(children) => { - let mut new_copy_into_location = self.clone(); - assert_eq!(children.len(), 1); - new_copy_into_location.input = children[0]; - Box::new(new_copy_into_location) - } - } + fn derive(&self, mut children: Vec>) -> Box { + let mut new_physical_plan = self.clone(); + assert_eq!(children.len(), 1); + new_physical_plan.input = children.pop().unwrap(); + Box::new(new_physical_plan) } } diff --git a/src/query/sql/src/executor/physical_plans/physical_copy_into_table.rs b/src/query/sql/src/executor/physical_plans/physical_copy_into_table.rs index 0494650a6722b..e5f56d1f3211a 100644 --- a/src/query/sql/src/executor/physical_plans/physical_copy_into_table.rs +++ b/src/query/sql/src/executor/physical_plans/physical_copy_into_table.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use databend_common_ast::ast::FormatTreeNode; use databend_common_catalog::plan::DataSourcePlan; use databend_common_catalog::plan::StageTableInfo; use databend_common_exception::Result; @@ -22,8 +23,9 @@ use databend_common_meta_app::schema::TableInfo; use databend_storages_common_table_meta::meta::TableMetaTimestamps; use enum_as_inner::EnumAsInner; +use crate::executor::format::FormatContext; +use crate::executor::physical_plan::DeriveHandle; use crate::executor::physical_plan::PhysicalPlan; -use crate::executor::physical_plan::PhysicalPlanDeriveHandle; use crate::executor::IPhysicalPlan; use crate::executor::PhysicalPlanMeta; use crate::plans::CopyIntoTableMode; @@ -77,36 +79,32 @@ impl IPhysicalPlan for CopyIntoTable { } } - fn derive_with( + fn to_format_node( &self, - handle: &mut Box, - ) -> Box { + _ctx: &mut FormatContext<'_>, + children: Vec>, + ) -> Result> { + Ok(FormatTreeNode::with_children( + format!("CopyIntoTable: {}", self.table_info), + children, + )) + } + + fn derive(&self, mut children: Vec>) -> Box { match &self.source { - CopyIntoTableSource::Query(query_physical_plan) => { - let derive_input = query_physical_plan.derive_with(handle); - match handle.derive(self, vec![derive_input]) { - Ok(v) => v, - Err(children) => { - let mut new_copy_into_table = self.clone(); - assert_eq!(children.len(), 1); - new_copy_into_table.source = - CopyIntoTableSource::Query(Box::new(children[0])); - Box::new(new_copy_into_table) - } - } + CopyIntoTableSource::Query(_) => { + let mut new_copy_into_table = self.clone(); + assert_eq!(children.len(), 1); + let input = children.pop().unwrap(); + new_copy_into_table.source = CopyIntoTableSource::Query(input); + Box::new(new_copy_into_table) } - CopyIntoTableSource::Stage(v) => { - let derive_input = v.derive_with(handle); - match handle.derive(self, vec![derive_input]) { - Ok(v) => v, - Err(children) => { - let mut new_copy_into_table = self.clone(); - assert_eq!(children.len(), 1); - new_copy_into_table.source = - CopyIntoTableSource::Stage(Box::new(children[0])); - Box::new(new_copy_into_table) - } - } + CopyIntoTableSource::Stage(_) => { + let mut new_copy_into_table = self.clone(); + assert_eq!(children.len(), 1); + let input = children.pop().unwrap(); + new_copy_into_table.source = CopyIntoTableSource::Stage(input); + Box::new(new_copy_into_table) } } } diff --git a/src/query/sql/src/executor/physical_plans/physical_distributed_insert_select.rs b/src/query/sql/src/executor/physical_plans/physical_distributed_insert_select.rs index aa4487eed9255..2a9373b08a2c0 100644 --- a/src/query/sql/src/executor/physical_plans/physical_distributed_insert_select.rs +++ b/src/query/sql/src/executor/physical_plans/physical_distributed_insert_select.rs @@ -18,7 +18,7 @@ use databend_common_expression::DataSchemaRef; use databend_common_meta_app::schema::TableInfo; use databend_storages_common_table_meta::meta::TableMetaTimestamps; -use crate::executor::physical_plan::PhysicalPlanDeriveHandle; +use crate::executor::physical_plan::DeriveHandle; use crate::executor::IPhysicalPlan; use crate::executor::PhysicalPlan; use crate::executor::PhysicalPlanMeta; @@ -65,20 +65,10 @@ impl IPhysicalPlan for DistributedInsertSelect { self.input.try_find_single_data_source() } - fn derive_with( - &self, - handle: &mut Box, - ) -> Box { - let derive_input = self.input.derive_with(handle); - - match handle.derive(self, vec![derive_input]) { - Ok(v) => v, - Err(children) => { - let mut new_distributed_insert_select = self.clone(); - assert_eq!(children.len(), 1); - new_distributed_insert_select.input = children[0]; - Box::new(new_distributed_insert_select) - } - } + fn derive(&self, mut children: Vec>) -> Box { + let mut new_physical_plan = self.clone(); + assert_eq!(children.len(), 1); + new_physical_plan.input = children.pop().unwrap(); + Box::new(new_physical_plan) } } diff --git a/src/query/sql/src/executor/physical_plans/physical_eval_scalar.rs b/src/query/sql/src/executor/physical_plans/physical_eval_scalar.rs index ff6d85db076e8..b23fe5ed6a1ce 100644 --- a/src/query/sql/src/executor/physical_plans/physical_eval_scalar.rs +++ b/src/query/sql/src/executor/physical_plans/physical_eval_scalar.rs @@ -17,6 +17,7 @@ use std::collections::HashMap; use std::collections::HashSet; use std::sync::Arc; +use databend_common_ast::ast::FormatTreeNode; use databend_common_catalog::plan::DataSourcePlan; use databend_common_exception::Result; use databend_common_expression::ConstantFolder; @@ -29,8 +30,11 @@ use databend_common_functions::BUILTIN_FUNCTIONS; use itertools::Itertools; use crate::executor::explain::PlanStatsInfo; +use crate::executor::format::format_output_columns; +use crate::executor::format::plan_stats_info_to_format_tree; +use crate::executor::format::FormatContext; +use crate::executor::physical_plan::DeriveHandle; use crate::executor::physical_plan::PhysicalPlan; -use crate::executor::physical_plan::PhysicalPlanDeriveHandle; use crate::executor::physical_plan_builder::PhysicalPlanBuilder; use crate::executor::IPhysicalPlan; use crate::executor::PhysicalPlanMeta; @@ -103,6 +107,43 @@ impl IPhysicalPlan for EvalScalar { Box::new(std::iter::once(&mut self.input)) } + fn to_format_node( + &self, + ctx: &mut FormatContext<'_>, + mut children: Vec>, + ) -> Result> { + if self.exprs.is_empty() { + assert_eq!(children.len(), 1); + return Ok(children.pop().unwrap()); + } + + let scalars = self + .exprs + .iter() + .map(|(expr, _)| expr.as_expr(&BUILTIN_FUNCTIONS).sql_display()) + .collect::>() + .join(", "); + + let mut node_children = vec![ + FormatTreeNode::new(format!( + "output columns: [{}]", + format_output_columns(self.output_schema()?, &ctx.metadata, true) + )), + FormatTreeNode::new(format!("expressions: [{scalars}]")), + ]; + + if let Some(info) = &self.stat_info { + node_children.extend(plan_stats_info_to_format_tree(info)); + } + + node_children.extend(children); + + Ok(FormatTreeNode::with_children( + "EvalScalar".to_string(), + node_children, + )) + } + #[recursive::recursive] fn try_find_single_data_source(&self) -> Option<&DataSourcePlan> { self.input.try_find_single_data_source() @@ -126,21 +167,11 @@ impl IPhysicalPlan for EvalScalar { )])) } - fn derive_with( - &self, - handle: &mut Box, - ) -> Box { - let derive_input = self.input.derive_with(handle); - - match handle.derive(self, vec![derive_input]) { - Ok(v) => v, - Err(children) => { - let mut new_eval_scalar = self.clone(); - assert_eq!(children.len(), 1); - new_eval_scalar.input = children[0]; - Box::new(new_eval_scalar) - } - } + fn derive(&self, mut children: Vec>) -> Box { + let mut new_physical_plan = self.clone(); + assert_eq!(children.len(), 1); + new_physical_plan.input = children.pop().unwrap(); + Box::new(new_physical_plan) } } diff --git a/src/query/sql/src/executor/physical_plans/physical_exchange.rs b/src/query/sql/src/executor/physical_plans/physical_exchange.rs index 4ce31edd54879..36dcaaaba7361 100644 --- a/src/query/sql/src/executor/physical_plans/physical_exchange.rs +++ b/src/query/sql/src/executor/physical_plans/physical_exchange.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use databend_common_ast::ast::FormatTreeNode; use databend_common_catalog::plan::DataSourcePlan; use databend_common_exception::Result; use databend_common_expression::ConstantFolder; @@ -19,7 +20,9 @@ use databend_common_expression::DataSchemaRef; use databend_common_expression::RemoteExpr; use databend_common_functions::BUILTIN_FUNCTIONS; -use crate::executor::physical_plan::PhysicalPlanDeriveHandle; +use crate::executor::format::format_output_columns; +use crate::executor::format::FormatContext; +use crate::executor::physical_plan::DeriveHandle; use crate::executor::physical_plans::common::FragmentKind; use crate::executor::IPhysicalPlan; use crate::executor::PhysicalPlan; @@ -49,11 +52,43 @@ impl IPhysicalPlan for Exchange { &mut self.meta } - fn children<'a>(&'a self) -> Box> + 'a> { + fn to_format_node( + &self, + ctx: &mut FormatContext<'_>, + children: Vec>, + ) -> Result> { + let mut node_children = vec![ + FormatTreeNode::new(format!( + "output columns: [{}]", + format_output_columns(self.output_schema()?, &ctx.metadata, true) + )), + FormatTreeNode::new(format!("exchange type: {}", match self.kind { + FragmentKind::Init => "Init-Partition".to_string(), + FragmentKind::Normal => format!( + "Hash({})", + self.keys + .iter() + .map(|key| { key.as_expr(&BUILTIN_FUNCTIONS).sql_display() }) + .collect::>() + .join(", ") + ), + FragmentKind::Expansive => "Broadcast".to_string(), + FragmentKind::Merge => "Merge".to_string(), + })), + ]; + + node_children.extend(children); + Ok(FormatTreeNode::with_children( + "Exchange".to_string(), + node_children, + )) + } + + fn children(&self) -> Box> + '_> { Box::new(std::iter::once(&self.input)) } - fn children_mut<'a>(&'a mut self) -> Box> + 'a> { + fn children_mut(&mut self) -> Box> + '_> { Box::new(std::iter::once(&mut self.input)) } @@ -66,21 +101,11 @@ impl IPhysicalPlan for Exchange { true } - fn derive_with( - &self, - handle: &mut Box, - ) -> Box { - let derive_input = self.input.derive_with(handle); - - match handle.derive(self, vec![derive_input]) { - Ok(v) => v, - Err(children) => { - let mut new_exchange = self.clone(); - assert_eq!(children.len(), 1); - new_exchange.input = children[0]; - Box::new(new_exchange) - } - } + fn derive(&self, mut children: Vec>) -> Box { + let mut new_physical_plan = self.clone(); + assert_eq!(children.len(), 1); + new_physical_plan.input = children.pop().unwrap(); + Box::new(new_physical_plan) } } @@ -99,7 +124,7 @@ impl PhysicalPlanBuilder { } // 2. Build physical plan. - let input = Box::new(self.build(s_expr.child(0)?, required).await?); + let input = self.build(s_expr.child(0)?, required).await?; let input_schema = input.output_schema()?; let mut keys = vec![]; let mut allow_adjust_parallelism = true; diff --git a/src/query/sql/src/executor/physical_plans/physical_exchange_sink.rs b/src/query/sql/src/executor/physical_plans/physical_exchange_sink.rs index 3006d1d2a0f4b..bef180b672a1c 100644 --- a/src/query/sql/src/executor/physical_plans/physical_exchange_sink.rs +++ b/src/query/sql/src/executor/physical_plans/physical_exchange_sink.rs @@ -12,12 +12,15 @@ // See the License for the specific language governing permissions and // limitations under the License. +use databend_common_ast::ast::FormatTreeNode; use databend_common_catalog::plan::DataSourcePlan; use databend_common_exception::Result; use databend_common_expression::DataSchemaRef; use databend_common_expression::RemoteExpr; -use crate::executor::physical_plan::PhysicalPlanDeriveHandle; +use crate::executor::format::format_output_columns; +use crate::executor::format::FormatContext; +use crate::executor::physical_plan::DeriveHandle; use crate::executor::physical_plans::common::FragmentKind; use crate::executor::IPhysicalPlan; use crate::executor::PhysicalPlan; @@ -65,6 +68,28 @@ impl IPhysicalPlan for ExchangeSink { Box::new(std::iter::once(&mut self.input)) } + fn to_format_node( + &self, + ctx: &mut FormatContext<'_>, + children: Vec>, + ) -> Result> { + let mut node_children = vec![FormatTreeNode::new(format!( + "output columns: [{}]", + format_output_columns(self.output_schema()?, &ctx.metadata, true) + ))]; + + node_children.push(FormatTreeNode::new(format!( + "destination fragment: [{}]", + self.destination_fragment_id + ))); + + node_children.extend(children); + Ok(FormatTreeNode::with_children( + "ExchangeSink".to_string(), + node_children, + )) + } + #[recursive::recursive] fn try_find_single_data_source(&self) -> Option<&DataSourcePlan> { self.input.try_find_single_data_source() @@ -74,20 +99,10 @@ impl IPhysicalPlan for ExchangeSink { true } - fn derive_with( - &self, - handle: &mut Box, - ) -> Box { - let derive_input = self.input.derive_with(handle); - - match handle.derive(self, vec![derive_input]) { - Ok(v) => v, - Err(children) => { - let mut new_exchange_sink = self.clone(); - assert_eq!(children.len(), 1); - new_exchange_sink.input = children[0]; - Box::new(new_exchange_sink) - } - } + fn derive(&self, mut children: Vec>) -> Box { + let mut new_physical_plan = self.clone(); + assert_eq!(children.len(), 1); + new_physical_plan.input = children.pop().unwrap(); + Box::new(new_physical_plan) } } diff --git a/src/query/sql/src/executor/physical_plans/physical_exchange_source.rs b/src/query/sql/src/executor/physical_plans/physical_exchange_source.rs index b0f1f94e7ec52..7e34b38ef378a 100644 --- a/src/query/sql/src/executor/physical_plans/physical_exchange_source.rs +++ b/src/query/sql/src/executor/physical_plans/physical_exchange_source.rs @@ -12,11 +12,14 @@ // See the License for the specific language governing permissions and // limitations under the License. +use databend_common_ast::ast::FormatTreeNode; use databend_common_catalog::plan::DataSourcePlan; use databend_common_exception::Result; use databend_common_expression::DataSchemaRef; -use crate::executor::physical_plan::PhysicalPlanDeriveHandle; +use crate::executor::format::format_output_columns; +use crate::executor::format::FormatContext; +use crate::executor::physical_plan::DeriveHandle; use crate::executor::IPhysicalPlan; use crate::executor::PhysicalPlanMeta; @@ -46,20 +49,34 @@ impl IPhysicalPlan for ExchangeSource { Ok(self.schema.clone()) } + fn to_format_node( + &self, + ctx: &mut FormatContext<'_>, + children: Vec>, + ) -> Result> { + let mut node_children = vec![FormatTreeNode::new(format!( + "output columns: [{}]", + format_output_columns(self.output_schema()?, &ctx.metadata, true) + ))]; + + node_children.push(FormatTreeNode::new(format!( + "source fragment: [{}]", + self.source_fragment_id + ))); + + node_children.extend(children); + Ok(FormatTreeNode::with_children( + "ExchangeSource".to_string(), + node_children, + )) + } + fn is_distributed_plan(&self) -> bool { true } - fn derive_with( - &self, - handle: &mut Box, - ) -> Box { - match handle.derive(self, vec![]) { - Ok(v) => v, - Err(children) => { - assert!(children.is_empty()); - Box::new(self.clone()) - } - } + fn derive(&self, children: Vec>) -> Box { + assert!(children.is_empty()); + Box::new(self.clone()) } } diff --git a/src/query/sql/src/executor/physical_plans/physical_expression_scan.rs b/src/query/sql/src/executor/physical_plans/physical_expression_scan.rs index 60ad789e678fc..0a642f82559fe 100644 --- a/src/query/sql/src/executor/physical_plans/physical_expression_scan.rs +++ b/src/query/sql/src/executor/physical_plans/physical_expression_scan.rs @@ -12,13 +12,17 @@ // See the License for the specific language governing permissions and // limitations under the License. +use databend_common_ast::ast::FormatTreeNode; use databend_common_exception::Result; use databend_common_expression::ConstantFolder; use databend_common_expression::DataSchemaRef; use databend_common_expression::RemoteExpr; use databend_common_functions::BUILTIN_FUNCTIONS; +use itertools::Itertools; -use crate::executor::physical_plan::PhysicalPlanDeriveHandle; +use crate::executor::format::format_output_columns; +use crate::executor::format::FormatContext; +use crate::executor::physical_plan::DeriveHandle; use crate::executor::IPhysicalPlan; use crate::executor::PhysicalPlan; use crate::executor::PhysicalPlanBuilder; @@ -59,21 +63,38 @@ impl IPhysicalPlan for ExpressionScan { Box::new(std::iter::once(&mut self.input)) } - fn derive_with( + fn to_format_node( &self, - handle: &mut Box, - ) -> Box { - let derive_input = self.input.derive_with(handle); - - match handle.derive(self, vec![derive_input]) { - Ok(v) => v, - Err(children) => { - let mut new_expression_scan = self.clone(); - assert_eq!(children.len(), 1); - new_expression_scan.input = children[0]; - Box::new(new_expression_scan) - } + ctx: &mut FormatContext<'_>, + children: Vec>, + ) -> Result> { + let mut node_children = Vec::with_capacity(self.values.len() + 1); + node_children.push(FormatTreeNode::new(format!( + "output columns: [{}]", + format_output_columns(self.output_schema()?, &ctx.metadata, true) + ))); + + for (i, value) in self.values.iter().enumerate() { + let column = value + .iter() + .map(|val| val.as_expr(&BUILTIN_FUNCTIONS).sql_display()) + .join(", "); + node_children.push(FormatTreeNode::new(format!("column {}: [{}]", i, column))); } + + node_children.extend(children); + + Ok(FormatTreeNode::with_children( + "ExpressionScan".to_string(), + node_children, + )) + } + + fn derive(&self, mut children: Vec>) -> Box { + let mut new_physical_plan = self.clone(); + assert_eq!(children.len(), 1); + new_physical_plan.input = children.pop().unwrap(); + Box::new(new_physical_plan) } } @@ -108,7 +129,7 @@ impl PhysicalPlanBuilder { Ok(Box::new(ExpressionScan { values, - input: Box::new(input), + input, output_schema: scan.schema.clone(), meta: PhysicalPlanMeta::new("ExpressionScan"), })) diff --git a/src/query/sql/src/executor/physical_plans/physical_filter.rs b/src/query/sql/src/executor/physical_plans/physical_filter.rs index fa8d87a66bef6..90dd2b716caa4 100644 --- a/src/query/sql/src/executor/physical_plans/physical_filter.rs +++ b/src/query/sql/src/executor/physical_plans/physical_filter.rs @@ -14,6 +14,7 @@ use std::collections::HashMap; +use databend_common_ast::ast::FormatTreeNode; use databend_common_catalog::plan::DataSourcePlan; use databend_common_exception::Result; use databend_common_expression::ConstantFolder; @@ -21,10 +22,14 @@ use databend_common_expression::DataSchemaRef; use databend_common_expression::DataSchemaRefExt; use databend_common_expression::RemoteExpr; use databend_common_functions::BUILTIN_FUNCTIONS; +use itertools::Itertools; use crate::executor::cast_expr_to_non_null_boolean; use crate::executor::explain::PlanStatsInfo; -use crate::executor::physical_plan::PhysicalPlanDeriveHandle; +use crate::executor::format::format_output_columns; +use crate::executor::format::plan_stats_info_to_format_tree; +use crate::executor::format::FormatContext; +use crate::executor::physical_plan::DeriveHandle; use crate::executor::IPhysicalPlan; use crate::executor::PhysicalPlan; use crate::executor::PhysicalPlanBuilder; @@ -76,6 +81,36 @@ impl IPhysicalPlan for Filter { Box::new(std::iter::once(&mut self.input)) } + fn to_format_node( + &self, + ctx: &mut FormatContext<'_>, + children: Vec>, + ) -> Result> { + let filter = self + .predicates + .iter() + .map(|pred| pred.as_expr(&BUILTIN_FUNCTIONS).sql_display()) + .join(", "); + + let mut node_children = vec![ + FormatTreeNode::new(format!( + "output columns: [{}]", + format_output_columns(self.output_schema()?, &ctx.metadata, true) + )), + FormatTreeNode::new(format!("filters: [{filter}]")), + ]; + + if let Some(info) = &self.stat_info { + node_children.extend(plan_stats_info_to_format_tree(info)); + } + + node_children.extend(children); + Ok(FormatTreeNode::with_children( + "Filter".to_string(), + node_children, + )) + } + #[recursive::recursive] fn try_find_single_data_source(&self) -> Option<&DataSourcePlan> { self.input.try_find_single_data_source() @@ -98,21 +133,11 @@ impl IPhysicalPlan for Filter { )])) } - fn derive_with( - &self, - handle: &mut Box, - ) -> Box { - let derive_input = self.input.derive_with(handle); - - match handle.derive(self, vec![derive_input]) { - Ok(v) => v, - Err(children) => { - let mut new_filter = self.clone(); - assert_eq!(children.len(), 1); - new_filter.input = children[0]; - Box::new(new_filter) - } - } + fn derive(&self, mut children: Vec>) -> Box { + let mut new_physical_plan = self.clone(); + assert_eq!(children.len(), 1); + new_physical_plan.input = children.pop().unwrap(); + Box::new(new_physical_plan) } } @@ -130,7 +155,7 @@ impl PhysicalPlanBuilder { }); // 2. Build physical plan. - let input = Box::new(self.build(s_expr.child(0)?, used).await?); + let input = self.build(s_expr.child(0)?, used).await?; required = required .union(self.metadata.read().get_retained_column()) .cloned() diff --git a/src/query/sql/src/executor/physical_plans/physical_hash_join.rs b/src/query/sql/src/executor/physical_plans/physical_hash_join.rs index 7be4725bd78dc..2fefc2c43a4c0 100644 --- a/src/query/sql/src/executor/physical_plans/physical_hash_join.rs +++ b/src/query/sql/src/executor/physical_plans/physical_hash_join.rs @@ -15,6 +15,7 @@ use std::collections::HashMap; use std::collections::HashSet; +use databend_common_ast::ast::FormatTreeNode; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_expression::type_check::check_cast; @@ -26,11 +27,15 @@ use databend_common_expression::DataSchemaRef; use databend_common_expression::DataSchemaRefExt; use databend_common_expression::RemoteExpr; use databend_common_functions::BUILTIN_FUNCTIONS; +use itertools::Itertools; use super::physical_join_filter::PhysicalRuntimeFilters; use super::JoinRuntimeFilter; use crate::executor::explain::PlanStatsInfo; -use crate::executor::physical_plan::PhysicalPlanDeriveHandle; +use crate::executor::format::format_output_columns; +use crate::executor::format::plan_stats_info_to_format_tree; +use crate::executor::format::FormatContext; +use crate::executor::physical_plan::DeriveHandle; use crate::executor::physical_plans::Exchange; use crate::executor::IPhysicalPlan; use crate::executor::PhysicalPlan; @@ -134,6 +139,115 @@ impl IPhysicalPlan for HashJoin { Box::new(std::iter::once(&mut self.probe).chain(std::iter::once(&mut self.build))) } + fn to_format_node( + &self, + ctx: &mut FormatContext<'_>, + mut children: Vec>, + ) -> Result> { + for rf in self.runtime_filter.filters.iter() { + ctx.scan_id_to_runtime_filters + .entry(rf.scan_id) + .or_default() + .push(rf.clone()); + } + + let build_keys = self + .build_keys + .iter() + .map(|scalar| scalar.as_expr(&BUILTIN_FUNCTIONS).sql_display()) + .collect::>() + .join(", "); + + let probe_keys = self + .probe_keys + .iter() + .map(|scalar| scalar.as_expr(&BUILTIN_FUNCTIONS).sql_display()) + .collect::>() + .join(", "); + + let is_null_equal = self.is_null_equal.iter().map(|b| format!("{b}")).join(", "); + + let filters = self + .non_equi_conditions + .iter() + .map(|filter| filter.as_expr(&BUILTIN_FUNCTIONS).sql_display()) + .collect::>() + .join(", "); + + assert_eq!(children.len(), 2); + children[0].payload = format!("{}(Build)", children[0].payload); + children[1].payload = format!("{}(Probe)", children[1].payload); + + let mut build_runtime_filters = vec![]; + for rf in self.runtime_filter.filters.iter() { + let mut s = format!( + "filter id:{}, build key:{}, probe key:{}, filter type:", + rf.id, + rf.build_key.as_expr(&BUILTIN_FUNCTIONS).sql_display(), + rf.probe_key.as_expr(&BUILTIN_FUNCTIONS).sql_display(), + ); + if rf.enable_bloom_runtime_filter { + s += "bloom,"; + } + if rf.enable_inlist_runtime_filter { + s += "inlist,"; + } + if rf.enable_min_max_runtime_filter { + s += "min_max,"; + } + s = s.trim_end_matches(',').to_string(); + build_runtime_filters.push(FormatTreeNode::new(s)); + } + + let mut node_children = vec![ + FormatTreeNode::new(format!( + "output columns: [{}]", + format_output_columns(self.output_schema()?, &ctx.metadata, true) + )), + FormatTreeNode::new(format!("join type: {}", self.join_type)), + FormatTreeNode::new(format!("build keys: [{build_keys}]")), + FormatTreeNode::new(format!("probe keys: [{probe_keys}]")), + FormatTreeNode::new(format!("keys is null equal: [{is_null_equal}]")), + FormatTreeNode::new(format!("filters: [{filters}]")), + ]; + + if !build_runtime_filters.is_empty() { + if self.broadcast_id.is_some() { + node_children.push(FormatTreeNode::with_children( + format!("build join filters(distributed):"), + build_runtime_filters, + )); + } else { + node_children.push(FormatTreeNode::with_children( + format!("build join filters:"), + build_runtime_filters, + )); + } + } + + if let Some((cache_index, column_map)) = &self.build_side_cache_info { + let mut column_indexes = column_map.keys().collect::>(); + column_indexes.sort(); + node_children.push(FormatTreeNode::new(format!("cache index: {}", cache_index))); + node_children.push(FormatTreeNode::new(format!( + "cache columns: {:?}", + column_indexes + ))); + } + + if let Some(info) = &self.stat_info { + let items = plan_stats_info_to_format_tree(info); + node_children.extend(items); + } + + node_children.extend(children); + + Ok(FormatTreeNode::with_children( + "HashJoin".to_string(), + node_children, + )) + } + fn get_desc(&self) -> Result { let mut conditions = self .build_keys @@ -194,23 +308,12 @@ impl IPhysicalPlan for HashJoin { Ok(labels) } - fn derive_with( - &self, - handle: &mut Box, - ) -> Box { - let derive_probe = self.probe.derive_with(handle); - let derive_build = self.build.derive_with(handle); - - match handle.derive(self, vec![derive_probe, derive_build]) { - Ok(v) => v, - Err(children) => { - let mut new_hash_join = self.clone(); - assert_eq!(children.len(), 2); - new_hash_join.probe = children[0]; - new_hash_join.build = children[1]; - Box::new(new_hash_join) - } - } + fn derive(&self, mut children: Vec>) -> Box { + let mut new_hash_join = self.clone(); + assert_eq!(children.len(), 2); + new_hash_join.build = children.pop().unwrap(); + new_hash_join.probe = children.pop().unwrap(); + Box::new(new_hash_join) } } @@ -222,8 +325,8 @@ impl PhysicalPlanBuilder { left_required: ColumnSet, right_required: ColumnSet, ) -> Result<(Box, Box)> { - let probe_side = Box::new(self.build(s_expr.child(0)?, left_required).await?); - let build_side = Box::new(self.build(s_expr.child(1)?, right_required).await?); + let probe_side = self.build(s_expr.child(0)?, left_required).await?; + let build_side = self.build(s_expr.child(1)?, right_required).await?; Ok((probe_side, build_side)) } diff --git a/src/query/sql/src/executor/physical_plans/physical_limit.rs b/src/query/sql/src/executor/physical_plans/physical_limit.rs index e9655c2917e71..65e8df9a20975 100644 --- a/src/query/sql/src/executor/physical_plans/physical_limit.rs +++ b/src/query/sql/src/executor/physical_plans/physical_limit.rs @@ -14,6 +14,7 @@ use std::collections::HashMap; +use databend_common_ast::ast::FormatTreeNode; use databend_common_catalog::plan::DataSourcePlan; use databend_common_exception::ErrorCode; use databend_common_exception::Result; @@ -22,7 +23,10 @@ use databend_common_expression::DataSchemaRef; use databend_common_expression::ROW_ID_COL_NAME; use crate::executor::explain::PlanStatsInfo; -use crate::executor::physical_plan::PhysicalPlanDeriveHandle; +use crate::executor::format::format_output_columns; +use crate::executor::format::plan_stats_info_to_format_tree; +use crate::executor::format::FormatContext; +use crate::executor::physical_plan::DeriveHandle; use crate::executor::physical_plans::physical_row_fetch::RowFetch; use crate::executor::IPhysicalPlan; use crate::executor::PhysicalPlan; @@ -63,6 +67,35 @@ impl IPhysicalPlan for Limit { Box::new(std::iter::once(&mut self.input)) } + fn to_format_node( + &self, + ctx: &mut FormatContext<'_>, + children: Vec>, + ) -> Result> { + let mut node_children = vec![ + FormatTreeNode::new(format!( + "output columns: [{}]", + format_output_columns(self.output_schema()?, &ctx.metadata, true) + )), + FormatTreeNode::new(format!( + "limit: {}", + self.limit + .map_or("NONE".to_string(), |limit| limit.to_string()) + )), + FormatTreeNode::new(format!("offset: {}", self.offset)), + ]; + + if let Some(info) = &self.stat_info { + node_children.extend(plan_stats_info_to_format_tree(info)); + } + + node_children.extend(children); + Ok(FormatTreeNode::with_children( + "Limit".to_string(), + node_children, + )) + } + #[recursive::recursive] fn try_find_single_data_source(&self) -> Option<&DataSourcePlan> { self.input.try_find_single_data_source() @@ -86,21 +119,11 @@ impl IPhysicalPlan for Limit { Ok(labels) } - fn derive_with( - &self, - handle: &mut Box, - ) -> Box { - let derive_input = self.input.derive_with(handle); - - match handle.derive(self, vec![derive_input]) { - Ok(v) => v, - Err(children) => { - let mut new_limit = self.clone(); - assert_eq!(children.len(), 1); - new_limit.input = children[0]; - Box::new(new_limit) - } - } + fn derive(&self, mut children: Vec>) -> Box { + let mut new_physical_plan = self.clone(); + assert_eq!(children.len(), 1); + new_physical_plan.input = children.pop().unwrap(); + Box::new(new_physical_plan) } } @@ -127,7 +150,7 @@ impl PhysicalPlanBuilder { let metadata = self.metadata.read().clone(); if limit.before_exchange || metadata.lazy_columns().is_empty() { return Ok(Box::new(Limit { - input: Box::new(input_plan), + input: input_plan, limit: limit.limit, offset: limit.offset, stat_info: Some(stat_info), @@ -147,7 +170,7 @@ impl PhysicalPlanBuilder { if !input_schema.has_field(&row_id_col_index.to_string()) { return Ok(Box::new(Limit { - input: Box::new(input_plan), + input: input_plan, limit: limit.limit, offset: limit.offset, stat_info: Some(stat_info), @@ -170,7 +193,7 @@ impl PhysicalPlanBuilder { if limit.before_exchange || lazy_columns.is_empty() { // If there is no lazy column, we don't need to build a `RowFetch` plan. return Ok(Box::new(Limit { - input: Box::new(input_plan), + input: input_plan, limit: limit.limit, offset: limit.offset, stat_info: Some(stat_info), @@ -210,7 +233,7 @@ impl PhysicalPlanBuilder { meta: PhysicalPlanMeta::new("RowFetch"), input: Box::new(Limit { meta: PhysicalPlanMeta::new("RowFetch"), - input: Box::new(input_plan), + input: input_plan, limit: limit.limit, offset: limit.offset, stat_info: Some(stat_info.clone()), diff --git a/src/query/sql/src/executor/physical_plans/physical_multi_table_insert.rs b/src/query/sql/src/executor/physical_plans/physical_multi_table_insert.rs index 17dc0b004128f..ac468ca33e846 100644 --- a/src/query/sql/src/executor/physical_plans/physical_multi_table_insert.rs +++ b/src/query/sql/src/executor/physical_plans/physical_multi_table_insert.rs @@ -14,16 +14,20 @@ use std::sync::Arc; +use databend_common_ast::ast::FormatTreeNode; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_expression::DataSchemaRef; use databend_common_expression::RemoteExpr; +use databend_common_functions::BUILTIN_FUNCTIONS; use databend_common_meta_app::schema::CatalogInfo; use databend_common_meta_app::schema::TableInfo; use databend_common_meta_app::schema::UpdateStreamMetaReq; use databend_storages_common_table_meta::meta::TableMetaTimestamps; +use itertools::Itertools; -use crate::executor::physical_plan::PhysicalPlanDeriveHandle; +use crate::executor::format::FormatContext; +use crate::executor::physical_plan::DeriveHandle; use crate::executor::IPhysicalPlan; use crate::executor::PhysicalPlan; use crate::executor::PhysicalPlanMeta; @@ -56,21 +60,29 @@ impl IPhysicalPlan for Duplicate { Box::new(std::iter::once(&mut self.input)) } - fn derive_with( + fn to_format_node( &self, - handle: &mut Box, - ) -> Box { - let derive_input = self.input.derive_with(handle); - - match handle.derive(self, vec![derive_input]) { - Ok(v) => v, - Err(children) => { - let mut new_duplicate = self.clone(); - assert_eq!(children.len(), 1); - new_duplicate.input = children[0]; - Box::new(new_duplicate) - } - } + _: &mut FormatContext<'_>, + children: Vec>, + ) -> Result> { + let mut node_children = vec![FormatTreeNode::new(format!( + "Duplicate data to {} branch", + self.n + ))]; + + node_children.extend(children); + + Ok(FormatTreeNode::with_children( + "Duplicate".to_string(), + node_children, + )) + } + + fn derive(&self, mut children: Vec>) -> Box { + let mut new_physical_plan = self.clone(); + assert_eq!(children.len(), 1); + new_physical_plan.input = children.pop().unwrap(); + Box::new(new_physical_plan) } } @@ -101,21 +113,21 @@ impl IPhysicalPlan for Shuffle { Box::new(std::iter::once(&mut self.input)) } - fn derive_with( + fn to_format_node( &self, - handle: &mut Box, - ) -> Box { - let derive_input = self.input.derive_with(handle); - - match handle.derive(self, vec![derive_input]) { - Ok(v) => v, - Err(children) => { - let mut new_shuffle = self.clone(); - assert_eq!(children.len(), 1); - new_shuffle.input = children[0]; - Box::new(new_shuffle) - } - } + _: &mut FormatContext<'_>, + mut children: Vec>, + ) -> Result> { + // ignore self + assert_eq!(children.len(), 1); + Ok(children.pop().unwrap()) + } + + fn derive(&self, mut children: Vec>) -> Box { + let mut new_physical_plan = self.clone(); + assert_eq!(children.len(), 1); + new_physical_plan.input = children.pop().unwrap(); + Box::new(new_physical_plan) } } @@ -173,21 +185,40 @@ impl IPhysicalPlan for ChunkFilter { Box::new(std::iter::once(&mut self.input)) } - fn derive_with( + fn to_format_node( &self, - handle: &mut Box, - ) -> Box { - let derive_input = self.input.derive_with(handle); - - match handle.derive(self, vec![derive_input]) { - Ok(v) => v, - Err(children) => { - let mut new_chunk_filter = self.clone(); - assert_eq!(children.len(), 1); - new_chunk_filter.input = children[0]; - Box::new(new_chunk_filter) + _: &mut FormatContext<'_>, + mut children: Vec>, + ) -> Result> { + if self.predicates.iter().all(|x| x.is_none()) { + assert_eq!(children.len(), 1); + return Ok(children.pop().unwrap()); + } + let mut node_children = Vec::new(); + for (i, predicate) in self.predicates.iter().enumerate() { + if let Some(predicate) = predicate { + node_children.push(FormatTreeNode::new(format!( + "branch {}: {}", + i, + predicate.as_expr(&BUILTIN_FUNCTIONS).sql_display() + ))); + } else { + node_children.push(FormatTreeNode::new(format!("branch {}: None", i))); } } + + node_children.extend(children); + Ok(FormatTreeNode::with_children( + "Filter".to_string(), + node_children, + )) + } + + fn derive(&self, mut children: Vec>) -> Box { + let mut new_physical_plan = self.clone(); + assert_eq!(children.len(), 1); + new_physical_plan.input = children.pop().unwrap(); + Box::new(new_physical_plan) } } @@ -218,21 +249,45 @@ impl IPhysicalPlan for ChunkEvalScalar { Box::new(std::iter::once(&mut self.input)) } - fn derive_with( + fn to_format_node( &self, - handle: &mut Box, - ) -> Box { - let derive_input = self.input.derive_with(handle); - - match handle.derive(self, vec![derive_input]) { - Ok(v) => v, - Err(children) => { - let mut new_chunk_eval_scalar = self.clone(); - assert_eq!(children.len(), 1); - new_chunk_eval_scalar.input = children[0]; - Box::new(new_chunk_eval_scalar) + _: &mut FormatContext<'_>, + mut children: Vec>, + ) -> Result> { + if self.eval_scalars.iter().all(|x| x.is_none()) { + assert_eq!(children.len(), 1); + return Ok(children.pop().unwrap()); + } + + let mut node_children = Vec::new(); + for (i, eval_scalar) in self.eval_scalars.iter().enumerate() { + if let Some(eval_scalar) = eval_scalar { + node_children.push(FormatTreeNode::new(format!( + "branch {}: {}", + i, + eval_scalar + .remote_exprs + .iter() + .map(|x| x.as_expr(&BUILTIN_FUNCTIONS).sql_display()) + .join(", ") + ))); + } else { + node_children.push(FormatTreeNode::new(format!("branch {}: None", i))); } } + + node_children.extend(children); + Ok(FormatTreeNode::with_children( + "EvalScalar".to_string(), + node_children, + )) + } + + fn derive(&self, mut children: Vec>) -> Box { + let mut new_physical_plan = self.clone(); + assert_eq!(children.len(), 1); + new_physical_plan.input = children.pop().unwrap(); + Box::new(new_physical_plan) } } @@ -269,21 +324,20 @@ impl IPhysicalPlan for ChunkCastSchema { Box::new(std::iter::once(&mut self.input)) } - fn derive_with( + fn to_format_node( &self, - handle: &mut Box, - ) -> Box { - let derive_input = self.input.derive_with(handle); - - match handle.derive(self, vec![derive_input]) { - Ok(v) => v, - Err(children) => { - let mut new_chunk_cast_schema = self.clone(); - assert_eq!(children.len(), 1); - new_chunk_cast_schema.input = children[0]; - Box::new(new_chunk_cast_schema) - } - } + _: &mut FormatContext<'_>, + mut children: Vec>, + ) -> Result> { + assert_eq!(children.len(), 1); + Ok(children.pop().unwrap()) + } + + fn derive(&self, mut children: Vec>) -> Box { + let mut new_physical_plan = self.clone(); + assert_eq!(children.len(), 1); + new_physical_plan.input = children.pop().unwrap(); + Box::new(new_physical_plan) } } @@ -320,21 +374,21 @@ impl IPhysicalPlan for ChunkFillAndReorder { Box::new(std::iter::once(&mut self.input)) } - fn derive_with( + fn to_format_node( &self, - handle: &mut Box, - ) -> Box { - let derive_input = self.input.derive_with(handle); - - match handle.derive(self, vec![derive_input]) { - Ok(v) => v, - Err(children) => { - let mut new_chunk_fill_and_reorder = self.clone(); - assert_eq!(children.len(), 1); - new_chunk_fill_and_reorder.input = children[0]; - Box::new(new_chunk_fill_and_reorder) - } - } + _: &mut FormatContext<'_>, + mut children: Vec>, + ) -> Result> { + // ignore self + assert_eq!(children.len(), 1); + Ok(children.pop().unwrap()) + } + + fn derive(&self, mut children: Vec>) -> Box { + let mut new_physical_plan = self.clone(); + assert_eq!(children.len(), 1); + new_physical_plan.input = children.pop().unwrap(); + Box::new(new_physical_plan) } } @@ -371,21 +425,22 @@ impl IPhysicalPlan for ChunkAppendData { Box::new(std::iter::once(&mut self.input)) } - fn derive_with( + fn to_format_node( &self, - handle: &mut Box, - ) -> Box { - let derive_input = self.input.derive_with(handle); - - match handle.derive(self, vec![derive_input]) { - Ok(v) => v, - Err(children) => { - let mut new_chunk_append_data = self.clone(); - assert_eq!(children.len(), 1); - new_chunk_append_data.input = children[0]; - Box::new(new_chunk_append_data) - } - } + _ctx: &mut FormatContext<'_>, + children: Vec>, + ) -> Result> { + Ok(FormatTreeNode::with_children( + "WriteData".to_string(), + children, + )) + } + + fn derive(&self, mut children: Vec>) -> Box { + let mut new_physical_plan = self.clone(); + assert_eq!(children.len(), 1); + new_physical_plan.input = children.pop().unwrap(); + Box::new(new_physical_plan) } } @@ -423,21 +478,21 @@ impl IPhysicalPlan for ChunkMerge { Box::new(std::iter::once(&mut self.input)) } - fn derive_with( + fn to_format_node( &self, - handle: &mut Box, - ) -> Box { - let derive_input = self.input.derive_with(handle); - - match handle.derive(self, vec![derive_input]) { - Ok(v) => v, - Err(children) => { - let mut new_chunk_merge = self.clone(); - assert_eq!(children.len(), 1); - new_chunk_merge.input = children[0]; - Box::new(new_chunk_merge) - } - } + _: &mut FormatContext<'_>, + mut children: Vec>, + ) -> Result> { + // ignore self + assert_eq!(children.len(), 1); + Ok(children.pop().unwrap()) + } + + fn derive(&self, mut children: Vec>) -> Box { + let mut new_physical_plan = self.clone(); + assert_eq!(children.len(), 1); + new_physical_plan.input = children.pop().unwrap(); + Box::new(new_physical_plan) } } @@ -471,20 +526,10 @@ impl IPhysicalPlan for ChunkCommitInsert { Box::new(std::iter::once(&mut self.input)) } - fn derive_with( - &self, - handle: &mut Box, - ) -> Box { - let derive_input = self.input.derive_with(handle); - - match handle.derive(self, vec![derive_input]) { - Ok(v) => v, - Err(children) => { - let mut new_chunk_commit_insert = self.clone(); - assert_eq!(children.len(), 1); - new_chunk_commit_insert.input = children[0]; - Box::new(new_chunk_commit_insert) - } - } + fn derive(&self, mut children: Vec>) -> Box { + let mut new_physical_plan = self.clone(); + assert_eq!(children.len(), 1); + new_physical_plan.input = children.pop().unwrap(); + Box::new(new_physical_plan) } } diff --git a/src/query/sql/src/executor/physical_plans/physical_mutation.rs b/src/query/sql/src/executor/physical_plans/physical_mutation.rs index 48a2aca6959f1..23f0bd69020ae 100644 --- a/src/query/sql/src/executor/physical_plans/physical_mutation.rs +++ b/src/query/sql/src/executor/physical_plans/physical_mutation.rs @@ -15,6 +15,7 @@ use std::collections::HashMap; use std::sync::Arc; +use databend_common_ast::ast::FormatTreeNode; use databend_common_catalog::plan::DataSourcePlan; use databend_common_catalog::plan::NUM_ROW_ID_PREFIX_BITS; use databend_common_catalog::table_context::TableContext; @@ -49,8 +50,9 @@ use super::CommitType; use crate::binder::wrap_cast; use crate::binder::MutationStrategy; use crate::binder::MutationType; +use crate::executor::format::FormatContext; +use crate::executor::physical_plan::DeriveHandle; use crate::executor::physical_plan::PhysicalPlan; -use crate::executor::physical_plan::PhysicalPlanDeriveHandle; use crate::executor::physical_plans::CommitSink; use crate::executor::physical_plans::Exchange; use crate::executor::physical_plans::FragmentKind; @@ -123,21 +125,31 @@ impl IPhysicalPlan for Mutation { Box::new(std::iter::once(&mut self.input)) } - fn derive_with( + fn to_format_node( &self, - handle: &mut Box, - ) -> Box { - let derive_input = self.input.derive_with(handle); - - match handle.derive(self, vec![derive_input]) { - Ok(v) => v, - Err(children) => { - let mut new_mutation = self.clone(); - assert_eq!(children.len(), 1); - new_mutation.input = children[0]; - Box::new(new_mutation) - } - } + ctx: &mut FormatContext<'_>, + children: Vec>, + ) -> Result> { + let table_entry = ctx.metadata.table(self.target_table_index).clone(); + let mut node_children = vec![FormatTreeNode::new(format!( + "target table: [catalog: {}] [database: {}] [table: {}]", + table_entry.catalog(), + table_entry.database(), + table_entry.name() + ))]; + + node_children.extend(children); + Ok(FormatTreeNode::with_children( + "DataMutation".to_string(), + node_children, + )) + } + + fn derive(&self, mut children: Vec>) -> Box { + let mut new_physical_plan = self.clone(); + assert_eq!(children.len(), 1); + new_physical_plan.input = children.pop().unwrap(); + Box::new(new_physical_plan) } } @@ -189,7 +201,7 @@ impl PhysicalPlanBuilder { if *truncate_table { // Do truncate. plan = Box::new(CommitSink { - input: Box::new(plan), + input: plan, snapshot: mutation_build_info.table_snapshot, table_info: table_info.clone(), // let's use update first, we will do some optimizations and select exact strategy @@ -323,7 +335,7 @@ impl PhysicalPlanBuilder { // into matched and not matched parts. if matches!(strategy, MutationStrategy::MixedMatched) { plan = Box::new(MutationSplit { - input: Box::new(plan), + input: plan, split_index: row_id_offset, meta: PhysicalPlanMeta::new("MutationSplit"), }); @@ -456,7 +468,7 @@ impl PhysicalPlanBuilder { } plan = Box::new(MutationManipulate { - input: Box::new(plan.clone()), + input: plan, table_info: table_info.clone(), unmatched: unmatched.clone(), matched: matched.clone(), @@ -466,10 +478,11 @@ impl PhysicalPlanBuilder { can_try_update_column_only: *can_try_update_column_only, unmatched_schema: mutation_input_schema.clone(), meta: PhysicalPlanMeta::new("MutationManipulate"), + target_table_index: *target_table_index, }); plan = Box::new(MutationOrganize { - input: Box::new(plan.clone()), + input: plan, strategy: strategy.clone(), meta: PhysicalPlanMeta::new("MutationOrganize"), }); @@ -496,7 +509,7 @@ impl PhysicalPlanBuilder { table_meta_timestamps: mutation_build_info.table_meta_timestamps, }); - if distributed { + if *distributed { plan = Box::new(Exchange { input: plan, kind: FragmentKind::Merge, diff --git a/src/query/sql/src/executor/physical_plans/physical_mutation_into_organize.rs b/src/query/sql/src/executor/physical_plans/physical_mutation_into_organize.rs index d70d739457159..7c761c6f1aca6 100644 --- a/src/query/sql/src/executor/physical_plans/physical_mutation_into_organize.rs +++ b/src/query/sql/src/executor/physical_plans/physical_mutation_into_organize.rs @@ -12,12 +12,14 @@ // See the License for the specific language governing permissions and // limitations under the License. +use databend_common_ast::ast::FormatTreeNode; use databend_common_exception::Result; use databend_common_expression::DataSchemaRef; use crate::binder::MutationStrategy; +use crate::executor::format::FormatContext; +use crate::executor::physical_plan::DeriveHandle; use crate::executor::physical_plan::PhysicalPlan; -use crate::executor::physical_plan::PhysicalPlanDeriveHandle; use crate::executor::IPhysicalPlan; use crate::executor::PhysicalPlanMeta; @@ -48,20 +50,20 @@ impl IPhysicalPlan for MutationOrganize { Box::new(std::iter::once(&mut self.input)) } - fn derive_with( + fn to_format_node( &self, - handle: &mut Box, - ) -> Box { - let derive_input = self.input.derive_with(handle); + _: &mut FormatContext<'_>, + mut children: Vec>, + ) -> Result> { + // ignore self + assert_eq!(children.len(), 1); + Ok(children.pop().unwrap()) + } - match handle.derive(self, vec![derive_input]) { - Ok(v) => v, - Err(children) => { - let mut new_mutation_organize = self.clone(); - assert_eq!(children.len(), 1); - new_mutation_organize.input = children[0]; - Box::new(new_mutation_organize) - } - } + fn derive(&self, mut children: Vec>) -> Box { + let mut new_physical_plan = self.clone(); + assert_eq!(children.len(), 1); + new_physical_plan.input = children.pop().unwrap(); + Box::new(new_physical_plan) } } diff --git a/src/query/sql/src/executor/physical_plans/physical_mutation_into_split.rs b/src/query/sql/src/executor/physical_plans/physical_mutation_into_split.rs index 86b1352ed6c07..f5ac2c9849c05 100644 --- a/src/query/sql/src/executor/physical_plans/physical_mutation_into_split.rs +++ b/src/query/sql/src/executor/physical_plans/physical_mutation_into_split.rs @@ -12,11 +12,13 @@ // See the License for the specific language governing permissions and // limitations under the License. +use databend_common_ast::ast::FormatTreeNode; use databend_common_exception::Result; use databend_common_expression::DataSchemaRef; +use crate::executor::format::FormatContext; +use crate::executor::physical_plan::DeriveHandle; use crate::executor::physical_plan::PhysicalPlan; -use crate::executor::physical_plan::PhysicalPlanDeriveHandle; use crate::executor::IPhysicalPlan; use crate::executor::PhysicalPlanMeta; use crate::IndexType; @@ -48,20 +50,20 @@ impl IPhysicalPlan for MutationSplit { Box::new(std::iter::once(&mut self.input)) } - fn derive_with( + fn to_format_node( &self, - handle: &mut Box, - ) -> Box { - let derive_input = self.input.derive_with(handle); + _: &mut FormatContext<'_>, + mut children: Vec>, + ) -> Result> { + // ignore self + assert_eq!(children.len(), 1); + Ok(children.pop().unwrap()) + } - match handle.derive(self, vec![derive_input]) { - Ok(v) => v, - Err(children) => { - let mut new_mutation_split = self.clone(); - assert_eq!(children.len(), 1); - new_mutation_split.input = children[0]; - Box::new(new_mutation_split) - } - } + fn derive(&self, mut children: Vec>) -> Box { + let mut new_physical_plan = self.clone(); + assert_eq!(children.len(), 1); + new_physical_plan.input = children.pop().unwrap(); + Box::new(new_physical_plan) } } diff --git a/src/query/sql/src/executor/physical_plans/physical_mutation_manipulate.rs b/src/query/sql/src/executor/physical_plans/physical_mutation_manipulate.rs index 6cc2de33b9fa8..1cbc7e46fe7de 100644 --- a/src/query/sql/src/executor/physical_plans/physical_mutation_manipulate.rs +++ b/src/query/sql/src/executor/physical_plans/physical_mutation_manipulate.rs @@ -14,15 +14,19 @@ use std::collections::HashMap; +use databend_common_ast::ast::FormatTreeNode; use databend_common_exception::Result; use databend_common_expression::DataSchemaRef; use databend_common_expression::FieldIndex; use databend_common_expression::RemoteExpr; +use databend_common_functions::BUILTIN_FUNCTIONS; use databend_common_meta_app::schema::TableInfo; +use itertools::Itertools; use crate::binder::MutationStrategy; +use crate::executor::format::FormatContext; +use crate::executor::physical_plan::DeriveHandle; use crate::executor::physical_plan::PhysicalPlan; -use crate::executor::physical_plan::PhysicalPlanDeriveHandle; use crate::executor::IPhysicalPlan; use crate::executor::PhysicalPlanMeta; @@ -44,6 +48,7 @@ pub struct MutationManipulate { pub row_id_idx: usize, pub can_try_update_column_only: bool, pub unmatched_schema: DataSchemaRef, + pub target_table_index: usize, } #[typetag::serde] @@ -66,20 +71,104 @@ impl IPhysicalPlan for MutationManipulate { Box::new(std::iter::once(&mut self.input)) } - fn derive_with( + fn to_format_node( &self, - handle: &mut Box, - ) -> Box { - let derive_input = self.input.derive_with(handle); - - match handle.derive(self, vec![derive_input]) { - Ok(v) => v, - Err(children) => { - let mut new_mutation_manipulate = self.clone(); - assert_eq!(children.len(), 1); - new_mutation_manipulate.input = children[0]; - Box::new(new_mutation_manipulate) + ctx: &mut FormatContext<'_>, + children: Vec>, + ) -> Result> { + let table_entry = ctx.metadata.table(self.target_table_index).clone(); + let target_schema = table_entry.table().schema_with_stream(); + + // Matched clauses. + let mut matched_children = Vec::with_capacity(self.matched.len()); + for evaluator in &self.matched { + let condition_format = evaluator.0.as_ref().map_or_else( + || "condition: None".to_string(), + |predicate| { + format!( + "condition: {}", + predicate.as_expr(&BUILTIN_FUNCTIONS).sql_display() + ) + }, + ); + + if evaluator.1.is_none() { + matched_children.push(FormatTreeNode::new(format!( + "matched delete: [{}]", + condition_format + ))); + } else { + let mut update_list = evaluator.1.as_ref().unwrap().clone(); + update_list.sort_by(|a, b| a.0.cmp(&b.0)); + let update_format = update_list + .iter() + .map(|(field_idx, expr)| { + format!( + "{} = {}", + target_schema.field(*field_idx).name(), + expr.as_expr(&BUILTIN_FUNCTIONS).sql_display() + ) + }) + .join(","); + matched_children.push(FormatTreeNode::new(format!( + "matched update: [{}, update set {}]", + condition_format, update_format + ))); } } + + // UnMatched clauses. + let mut unmatched_children = Vec::with_capacity(self.unmatched.len()); + for evaluator in &self.unmatched { + let condition_format = evaluator.1.as_ref().map_or_else( + || "condition: None".to_string(), + |predicate| { + format!( + "condition: {}", + predicate.as_expr(&BUILTIN_FUNCTIONS).sql_display() + ) + }, + ); + let insert_schema_format = evaluator + .0 + .fields + .iter() + .map(|field| field.name()) + .join(","); + + let values_format = evaluator + .2 + .iter() + .map(|expr| expr.as_expr(&BUILTIN_FUNCTIONS).sql_display()) + .join(","); + + let unmatched_format = format!( + "insert into ({}) values({})", + insert_schema_format, values_format + ); + + unmatched_children.push(FormatTreeNode::new(format!( + "unmatched insert: [{}, {}]", + condition_format, unmatched_format + ))); + } + + let mut node_children = vec![]; + + node_children.extend(matched_children); + node_children.extend(unmatched_children); + node_children.extend(children); + + Ok(FormatTreeNode::with_children( + "MutationManipulate".to_string(), + node_children, + )) + } + + fn derive(&self, mut children: Vec>) -> Box { + let mut new_physical_plan = self.clone(); + assert_eq!(children.len(), 1); + new_physical_plan.input = children.pop().unwrap(); + Box::new(new_physical_plan) } } diff --git a/src/query/sql/src/executor/physical_plans/physical_mutation_source.rs b/src/query/sql/src/executor/physical_plans/physical_mutation_source.rs index 1faa7276c4f3a..18cde5ce546d4 100644 --- a/src/query/sql/src/executor/physical_plans/physical_mutation_source.rs +++ b/src/query/sql/src/executor/physical_plans/physical_mutation_source.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use databend_common_ast::ast::FormatTreeNode; use databend_common_catalog::plan::DataSourcePlan; use databend_common_catalog::plan::Filters; use databend_common_catalog::plan::PartStatistics; @@ -29,7 +30,10 @@ use databend_common_meta_app::schema::TableInfo; use crate::binder::MutationType; use crate::executor::cast_expr_to_non_null_boolean; -use crate::executor::physical_plan::PhysicalPlanDeriveHandle; +use crate::executor::format::format_output_columns; +use crate::executor::format::part_stats_info_to_format_tree; +use crate::executor::format::FormatContext; +use crate::executor::physical_plan::DeriveHandle; use crate::executor::IPhysicalPlan; use crate::executor::PhysicalPlan; use crate::executor::PhysicalPlanBuilder; @@ -67,21 +71,51 @@ impl IPhysicalPlan for MutationSource { Ok(self.output_schema.clone()) } + fn to_format_node( + &self, + ctx: &mut FormatContext<'_>, + _: Vec>, + ) -> Result> { + let table = ctx.metadata.table(self.table_index); + let table_name = format!("{}.{}.{}", table.catalog(), table.database(), table.name()); + + let filters = self + .filters + .as_ref() + .map(|filters| filters.filter.as_expr(&BUILTIN_FUNCTIONS).sql_display()) + .unwrap_or_default(); + + let mut node_children = vec![ + FormatTreeNode::new(format!("table: {table_name}")), + FormatTreeNode::new(format!( + "output columns: [{}]", + format_output_columns(self.output_schema()?, ctx.metadata, false) + )), + FormatTreeNode::new(format!("filters: [{filters}]")), + ]; + + let payload = match self.input_type { + MutationType::Update => "Update", + MutationType::Delete if self.truncate_table => "DeleteAll", + MutationType::Delete => "Delete", + MutationType::Merge => "Merge", + }; + + // Part stats. + node_children.extend(part_stats_info_to_format_tree(&self.statistics)); + Ok(FormatTreeNode::with_children( + format!("MutationSource({})", payload), + node_children, + )) + } + fn try_find_mutation_source(&self) -> Option { Some(self.clone()) } - fn derive_with( - &self, - handle: &mut Box, - ) -> Box { - match handle.derive(self, vec![]) { - Ok(v) => v, - Err(children) => { - assert!(children.is_empty()); - Box::new(self.clone()) - } - } + fn derive(&self, children: Vec>) -> Box { + assert!(children.is_empty()); + Box::new(self.clone()) } } diff --git a/src/query/sql/src/executor/physical_plans/physical_project_set.rs b/src/query/sql/src/executor/physical_plans/physical_project_set.rs index dd20b5d91c8c9..a53732106a524 100644 --- a/src/query/sql/src/executor/physical_plans/physical_project_set.rs +++ b/src/query/sql/src/executor/physical_plans/physical_project_set.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use databend_common_ast::ast::FormatTreeNode; use databend_common_catalog::plan::DataSourcePlan; use databend_common_exception::Result; use databend_common_expression::ConstantFolder; @@ -23,7 +24,10 @@ use databend_common_functions::BUILTIN_FUNCTIONS; use itertools::Itertools; use crate::executor::explain::PlanStatsInfo; -use crate::executor::physical_plan::PhysicalPlanDeriveHandle; +use crate::executor::format::format_output_columns; +use crate::executor::format::plan_stats_info_to_format_tree; +use crate::executor::format::FormatContext; +use crate::executor::physical_plan::DeriveHandle; use crate::executor::IPhysicalPlan; use crate::executor::PhysicalPlan; use crate::executor::PhysicalPlanBuilder; @@ -81,6 +85,37 @@ impl IPhysicalPlan for ProjectSet { Box::new(std::iter::once(&mut self.input)) } + fn to_format_node( + &self, + ctx: &mut FormatContext<'_>, + children: Vec>, + ) -> Result> { + let mut node_children = vec![FormatTreeNode::new(format!( + "output columns: [{}]", + format_output_columns(self.output_schema()?, &ctx.metadata, true) + ))]; + + if let Some(info) = &self.stat_info { + let items = plan_stats_info_to_format_tree(info); + node_children.extend(items); + } + + node_children.extend(vec![FormatTreeNode::new(format!( + "set returning functions: {}", + self.srf_exprs + .iter() + .map(|(expr, _)| expr.clone().as_expr(&BUILTIN_FUNCTIONS).sql_display()) + .collect::>() + .join(", ") + ))]); + + node_children.extend(children); + Ok(FormatTreeNode::with_children( + "ProjectSet".to_string(), + node_children, + )) + } + #[recursive::recursive] fn try_find_single_data_source(&self) -> Option<&DataSourcePlan> { self.input.try_find_single_data_source() @@ -94,21 +129,11 @@ impl IPhysicalPlan for ProjectSet { .join(", ")) } - fn derive_with( - &self, - handle: &mut Box, - ) -> Box { - let derive_input = self.input.derive_with(handle); - - match handle.derive(self, vec![derive_input]) { - Ok(v) => v, - Err(children) => { - let mut new_project_set = self.clone(); - assert_eq!(children.len(), 1); - new_project_set.input = children[0]; - Box::new(new_project_set) - } - } + fn derive(&self, mut children: Vec>) -> Box { + let mut new_physical_plan = self.clone(); + assert_eq!(children.len(), 1); + new_physical_plan.input = children.pop().unwrap(); + Box::new(new_physical_plan) } } @@ -150,8 +175,8 @@ impl PhysicalPlanBuilder { } Ok(Box::new(ProjectSet { + input, meta: PhysicalPlanMeta::new("Unnest"), - input: Box::new(input), srf_exprs, projections, stat_info: Some(stat_info), diff --git a/src/query/sql/src/executor/physical_plans/physical_r_cte_scan.rs b/src/query/sql/src/executor/physical_plans/physical_r_cte_scan.rs index e59e6314f95f9..9dd806a500d97 100644 --- a/src/query/sql/src/executor/physical_plans/physical_r_cte_scan.rs +++ b/src/query/sql/src/executor/physical_plans/physical_r_cte_scan.rs @@ -19,7 +19,7 @@ use databend_common_expression::DataSchemaRef; use databend_common_expression::DataSchemaRefExt; use crate::executor::explain::PlanStatsInfo; -use crate::executor::physical_plan::PhysicalPlanDeriveHandle; +use crate::executor::physical_plan::DeriveHandle; use crate::executor::IPhysicalPlan; use crate::executor::PhysicalPlan; use crate::executor::PhysicalPlanBuilder; @@ -47,17 +47,9 @@ impl IPhysicalPlan for RecursiveCteScan { Ok(self.output_schema.clone()) } - fn derive_with( - &self, - handle: &mut Box, - ) -> Box { - match handle.derive(self, vec![]) { - Ok(v) => v, - Err(children) => { - assert!(children.is_empty()); - Box::new(self.clone()) - } - } + fn derive(&self, children: Vec>) -> Box { + assert!(children.is_empty()); + Box::new(self.clone()) } } diff --git a/src/query/sql/src/executor/physical_plans/physical_range_join.rs b/src/query/sql/src/executor/physical_plans/physical_range_join.rs index b3a9c81efe54b..ce4620df2d89a 100644 --- a/src/query/sql/src/executor/physical_plans/physical_range_join.rs +++ b/src/query/sql/src/executor/physical_plans/physical_range_join.rs @@ -14,6 +14,7 @@ use std::sync::Arc; +use databend_common_ast::ast::FormatTreeNode; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_expression::type_check::common_super_type; @@ -26,7 +27,10 @@ use databend_common_functions::BUILTIN_FUNCTIONS; use crate::binder::wrap_cast; use crate::binder::JoinPredicate; use crate::executor::explain::PlanStatsInfo; -use crate::executor::physical_plan::PhysicalPlanDeriveHandle; +use crate::executor::format::format_output_columns; +use crate::executor::format::plan_stats_info_to_format_tree; +use crate::executor::format::FormatContext; +use crate::executor::physical_plan::DeriveHandle; use crate::executor::IPhysicalPlan; use crate::executor::PhysicalPlan; use crate::executor::PhysicalPlanBuilder; @@ -83,6 +87,64 @@ impl IPhysicalPlan for RangeJoin { Box::new(std::iter::once(&mut self.left).chain(std::iter::once(&mut self.right))) } + fn to_format_node( + &self, + ctx: &mut FormatContext<'_>, + mut children: Vec>, + ) -> Result> { + let range_join_conditions = self + .conditions + .iter() + .map(|condition| { + let left = condition + .left_expr + .as_expr(&BUILTIN_FUNCTIONS) + .sql_display(); + let right = condition + .right_expr + .as_expr(&BUILTIN_FUNCTIONS) + .sql_display(); + format!("{left} {:?} {right}", condition.operator) + }) + .collect::>() + .join(", "); + + let other_conditions = self + .other_conditions + .iter() + .map(|filter| filter.as_expr(&BUILTIN_FUNCTIONS).sql_display()) + .collect::>() + .join(", "); + + assert_eq!(children.len(), 2); + children[0].payload = format!("{}(Left)", children[0].payload); + children[1].payload = format!("{}(Right)", children[1].payload); + + let mut node_children = vec![ + FormatTreeNode::new(format!( + "output columns: [{}]", + format_output_columns(self.output_schema()?, &ctx.metadata, true) + )), + FormatTreeNode::new(format!("join type: {}", self.join_type)), + FormatTreeNode::new(format!("range join conditions: [{range_join_conditions}]")), + FormatTreeNode::new(format!("other conditions: [{other_conditions}]")), + ]; + + if let Some(info) = &self.stat_info { + let items = plan_stats_info_to_format_tree(info); + node_children.extend(items); + } + + node_children.extend(children); + Ok(FormatTreeNode::with_children( + match self.range_join_type { + RangeJoinType::IEJoin => "IEJoin".to_string(), + RangeJoinType::Merge => "MergeJoin".to_string(), + }, + node_children, + )) + } + fn get_desc(&self) -> Result { let mut condition = self .conditions @@ -109,23 +171,12 @@ impl IPhysicalPlan for RangeJoin { Ok(condition.join(" AND ")) } - fn derive_with( - &self, - handle: &mut Box, - ) -> Box { - let derive_left = self.left.derive_with(handle); - let derive_right = self.right.derive_with(handle); - - match handle.derive(self, vec![derive_left, derive_right]) { - Ok(v) => v, - Err(children) => { - let mut new_range_join = self.clone(); - assert_eq!(children.len(), 2); - new_range_join.left = children[0]; - new_range_join.right = children[1]; - Box::new(new_range_join) - } - } + fn derive(&self, mut children: Vec>) -> Box { + let mut new_range_join = self.clone(); + assert_eq!(children.len(), 2); + new_range_join.right = children.pop().unwrap(); + new_range_join.left = children.pop().unwrap(); + Box::new(new_range_join) } } diff --git a/src/query/sql/src/executor/physical_plans/physical_recluster.rs b/src/query/sql/src/executor/physical_plans/physical_recluster.rs index 3995808abd150..d8d46d799f8ff 100644 --- a/src/query/sql/src/executor/physical_plans/physical_recluster.rs +++ b/src/query/sql/src/executor/physical_plans/physical_recluster.rs @@ -19,7 +19,7 @@ use databend_common_expression::DataSchemaRef; use databend_common_meta_app::schema::TableInfo; use databend_storages_common_table_meta::meta::TableMetaTimestamps; -use crate::executor::physical_plan::PhysicalPlanDeriveHandle; +use crate::executor::physical_plan::DeriveHandle; use crate::executor::IPhysicalPlan; use crate::executor::PhysicalPlan; use crate::executor::PhysicalPlanMeta; @@ -42,17 +42,9 @@ impl IPhysicalPlan for Recluster { &mut self.meta } - fn derive_with( - &self, - handle: &mut Box, - ) -> Box { - match handle.derive(self, vec![]) { - Ok(v) => v, - Err(children) => { - assert!(children.is_empty()); - Box::new(self.clone()) - } - } + fn derive(&self, children: Vec>) -> Box { + assert!(children.is_empty()); + Box::new(self.clone()) } } @@ -90,20 +82,10 @@ impl IPhysicalPlan for HilbertPartition { Box::new(std::iter::once(&mut self.input)) } - fn derive_with( - &self, - handle: &mut Box, - ) -> Box { - let derive_input = self.input.derive_with(handle); - - match handle.derive(self, vec![derive_input]) { - Ok(v) => v, - Err(children) => { - let mut new_hilbert_partition = self.clone(); - assert_eq!(children.len(), 1); - new_hilbert_partition.input = children[0]; - Box::new(new_hilbert_partition) - } - } + fn derive(&self, mut children: Vec>) -> Box { + let mut new_physical_plan = self.clone(); + assert_eq!(children.len(), 1); + new_physical_plan.input = children.pop().unwrap(); + Box::new(new_physical_plan) } } diff --git a/src/query/sql/src/executor/physical_plans/physical_replace_async_source.rs b/src/query/sql/src/executor/physical_plans/physical_replace_async_source.rs index 7e16a3ba47d5b..ac3cfda3d6626 100644 --- a/src/query/sql/src/executor/physical_plans/physical_replace_async_source.rs +++ b/src/query/sql/src/executor/physical_plans/physical_replace_async_source.rs @@ -15,7 +15,7 @@ use databend_common_catalog::plan::DataSourcePlan; use databend_common_expression::DataSchemaRef; -use crate::executor::physical_plan::PhysicalPlanDeriveHandle; +use crate::executor::physical_plan::DeriveHandle; use crate::executor::IPhysicalPlan; use crate::executor::PhysicalPlanMeta; use crate::plans::InsertValue; @@ -37,16 +37,8 @@ impl IPhysicalPlan for ReplaceAsyncSourcer { &mut self.meta } - fn derive_with( - &self, - handle: &mut Box, - ) -> Box { - match handle.derive(self, vec![]) { - Ok(v) => v, - Err(children) => { - assert!(children.is_empty()); - Box::new(self.clone()) - } - } + fn derive(&self, children: Vec>) -> Box { + assert!(children.is_empty()); + Box::new(self.clone()) } } diff --git a/src/query/sql/src/executor/physical_plans/physical_replace_deduplicate.rs b/src/query/sql/src/executor/physical_plans/physical_replace_deduplicate.rs index 28168f0a7906e..bb77e5644a59f 100644 --- a/src/query/sql/src/executor/physical_plans/physical_replace_deduplicate.rs +++ b/src/query/sql/src/executor/physical_plans/physical_replace_deduplicate.rs @@ -24,7 +24,7 @@ use databend_common_expression::TableSchemaRef; use databend_common_meta_app::schema::TableInfo; use databend_storages_common_table_meta::meta::ColumnStatistics; -use crate::executor::physical_plan::PhysicalPlanDeriveHandle; +use crate::executor::physical_plan::DeriveHandle; use crate::executor::physical_plans::common::OnConflictField; use crate::executor::IPhysicalPlan; use crate::executor::PhysicalPlan; @@ -70,21 +70,11 @@ impl IPhysicalPlan for ReplaceDeduplicate { Box::new(std::iter::once(&mut self.input)) } - fn derive_with( - &self, - handle: &mut Box, - ) -> Box { - let derive_input = self.input.derive_with(handle); - - match handle.derive(self, vec![derive_input]) { - Ok(v) => v, - Err(children) => { - let mut new_replace_deduplicate = self.clone(); - assert_eq!(children.len(), 1); - new_replace_deduplicate.input = children[0]; - Box::new(new_replace_deduplicate) - } - } + fn derive(&self, mut children: Vec>) -> Box { + let mut new_physical_plan = self.clone(); + assert_eq!(children.len(), 1); + new_physical_plan.input = children.pop().unwrap(); + Box::new(new_physical_plan) } } diff --git a/src/query/sql/src/executor/physical_plans/physical_replace_into.rs b/src/query/sql/src/executor/physical_plans/physical_replace_into.rs index 566ee4e4f41f4..1b4ef03fbc146 100644 --- a/src/query/sql/src/executor/physical_plans/physical_replace_into.rs +++ b/src/query/sql/src/executor/physical_plans/physical_replace_into.rs @@ -22,7 +22,7 @@ use databend_storages_common_table_meta::meta::BlockSlotDescription; use databend_storages_common_table_meta::meta::Location; use databend_storages_common_table_meta::meta::TableMetaTimestamps; -use crate::executor::physical_plan::PhysicalPlanDeriveHandle; +use crate::executor::physical_plan::DeriveHandle; use crate::executor::physical_plans::common::OnConflictField; use crate::executor::IPhysicalPlan; use crate::executor::PhysicalPlan; @@ -66,20 +66,10 @@ impl IPhysicalPlan for ReplaceInto { Box::new(std::iter::once(&mut self.input)) } - fn derive_with( - &self, - handle: &mut Box, - ) -> Box { - let derive_input = self.input.derive_with(handle); - - match handle.derive(self, vec![derive_input]) { - Ok(v) => v, - Err(children) => { - let mut new_replace_into = self.clone(); - assert_eq!(children.len(), 1); - new_replace_into.input = children[0]; - Box::new(new_replace_into) - } - } + fn derive(&self, mut children: Vec>) -> Box { + let mut new_physical_plan = self.clone(); + assert_eq!(children.len(), 1); + new_physical_plan.input = children.pop().unwrap(); + Box::new(new_physical_plan) } } diff --git a/src/query/sql/src/executor/physical_plans/physical_row_fetch.rs b/src/query/sql/src/executor/physical_plans/physical_row_fetch.rs index c2faeadc48080..4208f860ce92c 100644 --- a/src/query/sql/src/executor/physical_plans/physical_row_fetch.rs +++ b/src/query/sql/src/executor/physical_plans/physical_row_fetch.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use databend_common_ast::ast::FormatTreeNode; use databend_common_catalog::plan::DataSourcePlan; use databend_common_catalog::plan::Projection; use databend_common_exception::Result; @@ -21,7 +22,10 @@ use databend_common_expression::DataSchemaRefExt; use itertools::Itertools; use crate::executor::explain::PlanStatsInfo; -use crate::executor::physical_plan::PhysicalPlanDeriveHandle; +use crate::executor::format::format_output_columns; +use crate::executor::format::plan_stats_info_to_format_tree; +use crate::executor::format::FormatContext; +use crate::executor::physical_plan::DeriveHandle; use crate::executor::IPhysicalPlan; use crate::executor::PhysicalPlan; use crate::executor::PhysicalPlanMeta; @@ -68,6 +72,38 @@ impl IPhysicalPlan for RowFetch { Box::new(std::iter::once(&mut self.input)) } + fn to_format_node( + &self, + ctx: &mut FormatContext<'_>, + children: Vec>, + ) -> Result> { + let table_schema = self.source.source_info.schema(); + let projected_schema = self.cols_to_fetch.project_schema(&table_schema); + let fields_to_fetch = projected_schema.fields(); + + let mut node_children = vec![ + FormatTreeNode::new(format!( + "output columns: [{}]", + format_output_columns(self.output_schema()?, &ctx.metadata, true) + )), + FormatTreeNode::new(format!( + "columns to fetch: [{}]", + fields_to_fetch.iter().map(|f| f.name()).join(", ") + )), + ]; + + if let Some(info) = &self.stat_info { + node_children.extend(plan_stats_info_to_format_tree(info)); + } + + node_children.extend(children); + + Ok(FormatTreeNode::with_children( + "RowFetch".to_string(), + node_children, + )) + } + #[recursive::recursive] fn try_find_single_data_source(&self) -> Option<&DataSourcePlan> { self.input.try_find_single_data_source() @@ -79,20 +115,10 @@ impl IPhysicalPlan for RowFetch { Ok(projected_schema.fields.iter().map(|f| f.name()).join(", ")) } - fn derive_with( - &self, - handle: &mut Box, - ) -> Box { - let derive_input = self.input.derive_with(handle); - - match handle.derive(self, vec![derive_input]) { - Ok(v) => v, - Err(children) => { - let mut new_row_fetch = self.clone(); - assert_eq!(children.len(), 1); - new_row_fetch.input = children[0]; - Box::new(new_row_fetch) - } - } + fn derive(&self, mut children: Vec>) -> Box { + let mut new_physical_plan = self.clone(); + assert_eq!(children.len(), 1); + new_physical_plan.input = children.pop().unwrap(); + Box::new(new_physical_plan) } } diff --git a/src/query/sql/src/executor/physical_plans/physical_sort.rs b/src/query/sql/src/executor/physical_plans/physical_sort.rs index 35f194039bb7b..d0d191dd85415 100644 --- a/src/query/sql/src/executor/physical_plans/physical_sort.rs +++ b/src/query/sql/src/executor/physical_plans/physical_sort.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use databend_common_ast::ast::FormatTreeNode; use databend_common_catalog::plan::DataSourcePlan; use databend_common_exception::Result; use databend_common_expression::types::DataType; @@ -23,7 +24,10 @@ use databend_common_pipeline_transforms::processors::sort::utils::ORDER_COL_NAME use itertools::Itertools; use crate::executor::explain::PlanStatsInfo; -use crate::executor::physical_plan::PhysicalPlanDeriveHandle; +use crate::executor::format::format_output_columns; +use crate::executor::format::plan_stats_info_to_format_tree; +use crate::executor::format::FormatContext; +use crate::executor::physical_plan::DeriveHandle; use crate::executor::physical_plans::common::SortDesc; use crate::executor::physical_plans::WindowPartition; use crate::executor::physical_plans::WindowPartitionTopN; @@ -111,6 +115,48 @@ impl IPhysicalPlan for Sort { Box::new(std::iter::once(&mut self.input)) } + fn to_format_node( + &self, + ctx: &mut FormatContext<'_>, + children: Vec>, + ) -> Result> { + let sort_keys = self + .order_by + .iter() + .map(|sort_key| { + Ok(format!( + "{} {} {}", + sort_key.display_name, + if sort_key.asc { "ASC" } else { "DESC" }, + if sort_key.nulls_first { + "NULLS FIRST" + } else { + "NULLS LAST" + } + )) + }) + .collect::>>()? + .join(", "); + + let mut node_children = vec![ + FormatTreeNode::new(format!( + "output columns: [{}]", + format_output_columns(self.output_schema()?, &ctx.metadata, true) + )), + FormatTreeNode::new(format!("sort keys: [{sort_keys}]")), + ]; + + if let Some(info) = &self.stat_info { + node_children.extend(plan_stats_info_to_format_tree(info)); + } + + node_children.extend(children); + Ok(FormatTreeNode::with_children( + "Sort".to_string(), + node_children, + )) + } + #[recursive::recursive] fn try_find_single_data_source(&self) -> Option<&DataSourcePlan> { self.input.try_find_single_data_source() @@ -131,21 +177,11 @@ impl IPhysicalPlan for Sort { .join(", ")) } - fn derive_with( - &self, - handle: &mut Box, - ) -> Box { - let derive_input = self.input.derive_with(handle); - - match handle.derive(self, vec![derive_input]) { - Ok(v) => v, - Err(children) => { - let mut new_sort = self.clone(); - assert_eq!(children.len(), 1); - new_sort.input = children[0]; - Box::new(new_sort) - } - } + fn derive(&self, mut children: Vec>) -> Box { + let mut new_physical_plan = self.clone(); + assert_eq!(children.len(), 1); + new_physical_plan.input = children.pop().unwrap(); + Box::new(new_physical_plan) } } @@ -171,7 +207,7 @@ impl PhysicalPlanBuilder { sort: &crate::plans::Sort, mut required: ColumnSet, stat_info: PlanStatsInfo, - ) -> Result { + ) -> Result> { // 1. Prune unused Columns. sort.items.iter().for_each(|s| { required.insert(s.index); @@ -206,7 +242,7 @@ impl PhysicalPlanBuilder { return Ok(Box::new(WindowPartition { meta: PhysicalPlanMeta::new("WindowPartition"), - input: Box::new(input_plan.clone()), + input: input_plan.clone(), partition_by: window_partition.clone(), order_by: order_by.clone(), after_exchange: sort.after_exchange, @@ -227,7 +263,7 @@ impl PhysicalPlanBuilder { Ok(Box::new(Sort { order_by, pre_projection, - input: Box::new(input_plan), + input: input_plan, limit: sort.limit, after_exchange: sort.after_exchange, stat_info: Some(stat_info), diff --git a/src/query/sql/src/executor/physical_plans/physical_table_scan.rs b/src/query/sql/src/executor/physical_plans/physical_table_scan.rs index 92480efdf2515..b2b205619f4e2 100644 --- a/src/query/sql/src/executor/physical_plans/physical_table_scan.rs +++ b/src/query/sql/src/executor/physical_plans/physical_table_scan.rs @@ -18,6 +18,7 @@ use std::collections::HashMap; use std::collections::HashSet; use std::sync::Arc; +use databend_common_ast::ast::FormatTreeNode; use databend_common_ast::parser::token::TokenKind; use databend_common_ast::parser::tokenize_sql; use databend_common_catalog::catalog::CatalogManager; @@ -48,6 +49,7 @@ use databend_common_expression::TableSchema; use databend_common_expression::TableSchemaRef; use databend_common_expression::ROW_ID_COL_NAME; use databend_common_functions::BUILTIN_FUNCTIONS; +use itertools::Itertools; use jsonb::keypath::KeyPath; use jsonb::keypath::KeyPaths; use rand::distributions::Bernoulli; @@ -57,7 +59,11 @@ use rand::thread_rng; use crate::binder::INTERNAL_COLUMN_FACTORY; use crate::executor::cast_expr_to_non_null_boolean; use crate::executor::explain::PlanStatsInfo; -use crate::executor::physical_plan::PhysicalPlanDeriveHandle; +use crate::executor::format::format_output_columns; +use crate::executor::format::part_stats_info_to_format_tree; +use crate::executor::format::plan_stats_info_to_format_tree; +use crate::executor::format::FormatContext; +use crate::executor::physical_plan::DeriveHandle; use crate::executor::physical_plans::AddStreamColumn; use crate::executor::table_read_plan::ToReadDataSourcePlan; use crate::executor::IPhysicalPlan; @@ -123,6 +129,136 @@ impl IPhysicalPlan for TableScan { Ok(DataSchemaRefExt::create(fields)) } + fn to_format_node( + &self, + ctx: &mut FormatContext<'_>, + _: Vec>, + ) -> Result> { + if self.table_index == Some(DUMMY_TABLE_INDEX) { + return Ok(FormatTreeNode::new("DummyTableScan".to_string())); + } + + let table_name = match self.table_index { + None => format!( + "{}.{}", + self.source.source_info.catalog_name(), + self.source.source_info.desc() + ), + Some(table_index) => { + let table = ctx.metadata.table(table_index).clone(); + format!("{}.{}.{}", table.catalog(), table.database(), table.name()) + } + }; + let filters = self + .source + .push_downs + .as_ref() + .and_then(|extras| { + extras + .filters + .as_ref() + .map(|filters| filters.filter.as_expr(&BUILTIN_FUNCTIONS).sql_display()) + }) + .unwrap_or_default(); + + let limit = self + .source + .push_downs + .as_ref() + .map_or("NONE".to_string(), |extras| { + extras + .limit + .map_or("NONE".to_string(), |limit| limit.to_string()) + }); + + let virtual_columns = self.source.push_downs.as_ref().and_then(|extras| { + extras.virtual_column.as_ref().map(|virtual_column| { + let mut names = virtual_column + .virtual_column_fields + .iter() + .map(|c| c.name.clone()) + .collect::>(); + names.sort(); + names.iter().join(", ") + }) + }); + + let agg_index = self + .source + .push_downs + .as_ref() + .and_then(|extras| extras.agg_index.as_ref()); + + let mut children = vec![ + FormatTreeNode::new(format!("table: {table_name}")), + FormatTreeNode::new(format!( + "output columns: [{}]", + format_output_columns(self.output_schema()?, ctx.metadata, false) + )), + ]; + + // Part stats. + children.extend(part_stats_info_to_format_tree(&self.source.statistics)); + // Push downs. + let push_downs = format!("push downs: [filters: [{filters}], limit: {limit}]"); + children.push(FormatTreeNode::new(push_downs)); + + // runtime filters + let rf = ctx.scan_id_to_runtime_filters.get(&self.scan_id); + if let Some(rf) = rf { + let rf = rf.iter().map(|rf| format!("#{:?}", rf.id)).join(", "); + children.push(FormatTreeNode::new(format!("apply join filters: [{rf}]"))); + } + + // Virtual columns. + if let Some(virtual_columns) = virtual_columns { + if !virtual_columns.is_empty() { + let virtual_columns = format!("virtual columns: [{virtual_columns}]"); + children.push(FormatTreeNode::new(virtual_columns)); + } + } + + // Aggregating index + if let Some(agg_index) = agg_index { + let (_, agg_index_sql, _) = ctx + .metadata + .get_agg_indexes(&table_name) + .unwrap() + .iter() + .find(|(index, _, _)| *index == agg_index.index_id) + .unwrap(); + + children.push(FormatTreeNode::new(format!( + "aggregating index: [{agg_index_sql}]" + ))); + + let agg_sel = agg_index + .selection + .iter() + .map(|(expr, _)| expr.as_expr(&BUILTIN_FUNCTIONS).sql_display()) + .join(", "); + let agg_filter = agg_index + .filter + .as_ref() + .map(|f| f.as_expr(&BUILTIN_FUNCTIONS).sql_display()); + let text = if let Some(f) = agg_filter { + format!("rewritten query: [selection: [{agg_sel}], filter: {f}]") + } else { + format!("rewritten query: [selection: [{agg_sel}]]") + }; + children.push(FormatTreeNode::new(text)); + } + + if let Some(info) = &self.stat_info { + children.extend(plan_stats_info_to_format_tree(info)); + } + + Ok(FormatTreeNode::with_children( + "TableScan".to_string(), + children, + )) + } + fn try_find_single_data_source(&self) -> Option<&DataSourcePlan> { Some(&self.source) } @@ -175,17 +311,9 @@ impl IPhysicalPlan for TableScan { ])) } - fn derive_with( - &self, - handle: &mut Box, - ) -> Box { - match handle.derive(self, vec![]) { - Ok(v) => v, - Err(children) => { - assert!(children.is_empty()); - Box::new(self.clone()) - } - } + fn derive(&self, children: Vec>) -> Box { + assert!(children.is_empty()); + Box::new(self.clone()) } } diff --git a/src/query/sql/src/executor/physical_plans/physical_udf.rs b/src/query/sql/src/executor/physical_plans/physical_udf.rs index 0abddc607dbf2..135f1a3cfcb44 100644 --- a/src/query/sql/src/executor/physical_plans/physical_udf.rs +++ b/src/query/sql/src/executor/physical_plans/physical_udf.rs @@ -14,6 +14,7 @@ use std::collections::BTreeMap; +use databend_common_ast::ast::FormatTreeNode; use databend_common_catalog::plan::DataSourcePlan; use databend_common_exception::ErrorCode; use databend_common_exception::Result; @@ -25,7 +26,10 @@ use databend_common_functions::BUILTIN_FUNCTIONS; use itertools::Itertools; use crate::executor::explain::PlanStatsInfo; -use crate::executor::physical_plan::PhysicalPlanDeriveHandle; +use crate::executor::format::format_output_columns; +use crate::executor::format::plan_stats_info_to_format_tree; +use crate::executor::format::FormatContext; +use crate::executor::physical_plan::DeriveHandle; use crate::executor::IPhysicalPlan; use crate::executor::PhysicalPlan; use crate::executor::PhysicalPlanBuilder; @@ -34,6 +38,7 @@ use crate::optimizer::ir::SExpr; use crate::plans::UDFType; use crate::ColumnSet; use crate::IndexType; +use crate::Metadata; use crate::ScalarExpr; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] @@ -77,6 +82,40 @@ impl IPhysicalPlan for Udf { Box::new(std::iter::once(&mut self.input)) } + fn to_format_node( + &self, + ctx: &mut FormatContext<'_>, + children: Vec>, + ) -> Result> { + let mut node_children = vec![FormatTreeNode::new(format!( + "output columns: [{}]", + format_output_columns(self.output_schema()?, &ctx.metadata, true) + ))]; + + if let Some(info) = &self.stat_info { + let items = plan_stats_info_to_format_tree(info); + node_children.extend(items); + } + + node_children.extend(vec![FormatTreeNode::new(format!( + "udf functions: {}", + self.udf_funcs + .iter() + .map(|func| { + let arg_exprs = func.arg_exprs.join(", "); + format!("{}({})", func.func_name, arg_exprs) + }) + .collect::>() + .join(", ") + ))]); + + node_children.extend(children); + Ok(FormatTreeNode::with_children( + "Udf".to_string(), + node_children, + )) + } + #[recursive::recursive] fn try_find_single_data_source(&self) -> Option<&DataSourcePlan> { self.input.try_find_single_data_source() @@ -90,21 +129,11 @@ impl IPhysicalPlan for Udf { .join(", ")) } - fn derive_with( - &self, - handle: &mut Box, - ) -> Box { - let derive_input = self.input.derive_with(handle); - - match handle.derive(self, vec![derive_input]) { - Ok(v) => v, - Err(children) => { - let mut new_udf = self.clone(); - assert_eq!(children.len(), 1); - new_udf.input = children[0]; - Box::new(new_udf) - } - } + fn derive(&self, mut children: Vec>) -> Box { + let mut new_physical_plan = self.clone(); + assert_eq!(children.len(), 1); + new_physical_plan.input = children.pop().unwrap(); + Box::new(new_physical_plan) } } @@ -200,7 +229,7 @@ impl PhysicalPlanBuilder { .collect::>>()?; Ok(Box::new(Udf { - input: Box::new(input), + input, udf_funcs, script_udf: udf_plan.script_udf, stat_info: Some(stat_info), diff --git a/src/query/sql/src/executor/physical_plans/physical_union_all.rs b/src/query/sql/src/executor/physical_plans/physical_union_all.rs index 5dff25d4cce82..a8eb52d30cb07 100644 --- a/src/query/sql/src/executor/physical_plans/physical_union_all.rs +++ b/src/query/sql/src/executor/physical_plans/physical_union_all.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use databend_common_ast::ast::FormatTreeNode; use databend_common_exception::Result; use databend_common_expression::DataField; use databend_common_expression::DataSchema; @@ -21,7 +22,10 @@ use databend_common_expression::RemoteExpr; use itertools::Itertools; use crate::executor::explain::PlanStatsInfo; -use crate::executor::physical_plan::PhysicalPlanDeriveHandle; +use crate::executor::format::format_output_columns; +use crate::executor::format::plan_stats_info_to_format_tree; +use crate::executor::format::FormatContext; +use crate::executor::physical_plan::DeriveHandle; use crate::executor::IPhysicalPlan; use crate::executor::PhysicalPlan; use crate::executor::PhysicalPlanBuilder; @@ -70,6 +74,31 @@ impl IPhysicalPlan for UnionAll { Box::new(std::iter::once(&mut self.left).chain(std::iter::once(&mut self.right))) } + fn to_format_node( + &self, + ctx: &mut FormatContext<'_>, + children: Vec>, + ) -> Result> { + let mut node_children = vec![FormatTreeNode::new(format!( + "output columns: [{}]", + format_output_columns(self.output_schema()?, &ctx.metadata, true) + ))]; + + if let Some(info) = &self.stat_info { + let items = plan_stats_info_to_format_tree(info); + node_children.extend(items); + } + + let root = if !self.cte_scan_names.is_empty() { + "UnionAll(recursive cte)".to_string() + } else { + "UnionAll".to_string() + }; + + node_children.extend(children); + Ok(FormatTreeNode::with_children(root, node_children)) + } + fn get_desc(&self) -> Result { Ok(self .left_outputs @@ -79,23 +108,12 @@ impl IPhysicalPlan for UnionAll { .join(", ")) } - fn derive_with( - &self, - handle: &mut Box, - ) -> Box { - let derive_left = self.left.derive_with(handle); - let derive_right = self.right.derive_with(handle); - - match handle.derive(self, vec![derive_left, derive_right]) { - Ok(v) => v, - Err(children) => { - let mut new_union_all = self.clone(); - assert_eq!(children.len(), 2); - new_union_all.left = children[0]; - new_union_all.right = children[1]; - Box::new(new_union_all) - } - } + fn derive(&self, mut children: Vec>) -> Box { + let mut new_union_all = self.clone(); + assert_eq!(children.len(), 2); + new_union_all.right = children.pop().unwrap(); + new_union_all.left = children.pop().unwrap(); + Box::new(new_union_all) } } diff --git a/src/query/sql/src/executor/physical_plans/physical_window.rs b/src/query/sql/src/executor/physical_plans/physical_window.rs index 1e4dfd95c75fa..da5ee8bf5a51b 100644 --- a/src/query/sql/src/executor/physical_plans/physical_window.rs +++ b/src/query/sql/src/executor/physical_plans/physical_window.rs @@ -14,6 +14,7 @@ use std::fmt::Display; +use databend_common_ast::ast::FormatTreeNode; use databend_common_catalog::plan::DataSourcePlan; use databend_common_exception::ErrorCode; use databend_common_exception::Result; @@ -33,7 +34,10 @@ use databend_common_functions::BUILTIN_FUNCTIONS; use crate::binder::wrap_cast; use crate::executor::explain::PlanStatsInfo; -use crate::executor::physical_plan::PhysicalPlanDeriveHandle; +use crate::executor::format::format_output_columns; +use crate::executor::format::pretty_display_agg_desc; +use crate::executor::format::FormatContext; +use crate::executor::physical_plan::DeriveHandle; use crate::executor::physical_plans::common::AggregateFunctionDesc; use crate::executor::physical_plans::common::AggregateFunctionSignature; use crate::executor::physical_plans::common::SortDesc; @@ -93,6 +97,54 @@ impl IPhysicalPlan for Window { Box::new(std::iter::once(&mut self.input)) } + fn to_format_node( + &self, + ctx: &mut FormatContext<'_>, + children: Vec>, + ) -> Result> { + let partition_by = self + .partition_by + .iter() + .map(|&index| Ok(ctx.metadata.column(index).name())) + .collect::>>()? + .join(", "); + + let order_by = self + .order_by + .iter() + .map(|v| v.display_name.clone()) + .collect::>() + .join(", "); + + let frame = self.window_frame.to_string(); + + let func = match &self.func { + WindowFunction::Aggregate(agg) => pretty_display_agg_desc(agg, &ctx.metadata), + func => format!("{}", func), + }; + + let mut node_children = vec![ + FormatTreeNode::new(format!( + "output columns: [{}]", + format_output_columns(self.output_schema()?, &ctx.metadata, true) + )), + FormatTreeNode::new(format!("aggregate function: [{func}]")), + FormatTreeNode::new(format!("partition by: [{partition_by}]")), + FormatTreeNode::new(format!("order by: [{order_by}]")), + FormatTreeNode::new(format!("frame: [{frame}]")), + ]; + + if let Some(limit) = self.limit { + node_children.push(FormatTreeNode::new(format!("limit: [{limit}]"))) + } + + node_children.extend(children); + Ok(FormatTreeNode::with_children( + "Window".to_string(), + node_children, + )) + } + #[recursive::recursive] fn try_find_single_data_source(&self) -> Option<&DataSourcePlan> { self.input.try_find_single_data_source() @@ -126,21 +178,11 @@ impl IPhysicalPlan for Window { )) } - fn derive_with( - &self, - handle: &mut Box, - ) -> Box { - let derive_input = self.input.derive_with(handle); - - match handle.derive(self, vec![derive_input]) { - Ok(v) => v, - Err(children) => { - let mut new_window = self.clone(); - assert_eq!(children.len(), 1); - new_window.input = children[0]; - Box::new(new_window) - } - } + fn derive(&self, mut children: Vec>) -> Box { + let mut new_physical_plan = self.clone(); + assert_eq!(children.len(), 1); + new_physical_plan.input = children.pop().unwrap(); + Box::new(new_physical_plan) } } @@ -433,8 +475,8 @@ impl PhysicalPlanBuilder { }; Ok(Box::new(Window { + input, index: w.index, - input: Box::new(input), func, partition_by: partition_items, order_by: order_by_items, diff --git a/src/query/sql/src/executor/physical_plans/physical_window_partition.rs b/src/query/sql/src/executor/physical_plans/physical_window_partition.rs index e15837a8b723a..0939f27ad1545 100644 --- a/src/query/sql/src/executor/physical_plans/physical_window_partition.rs +++ b/src/query/sql/src/executor/physical_plans/physical_window_partition.rs @@ -12,12 +12,16 @@ // See the License for the specific language governing permissions and // limitations under the License. +use databend_common_ast::ast::FormatTreeNode; use databend_common_catalog::plan::DataSourcePlan; use databend_common_exception::Result; use databend_common_expression::DataSchemaRef; use crate::executor::explain::PlanStatsInfo; -use crate::executor::physical_plan::PhysicalPlanDeriveHandle; +use crate::executor::format::format_output_columns; +use crate::executor::format::plan_stats_info_to_format_tree; +use crate::executor::format::FormatContext; +use crate::executor::physical_plan::DeriveHandle; use crate::executor::physical_plans::SortDesc; use crate::executor::IPhysicalPlan; use crate::executor::PhysicalPlan; @@ -56,26 +60,51 @@ impl IPhysicalPlan for WindowPartition { Box::new(std::iter::once(&mut self.input)) } + fn to_format_node( + &self, + ctx: &mut FormatContext<'_>, + children: Vec>, + ) -> Result> { + let partition_by = self + .partition_by + .iter() + .map(|&index| Ok(ctx.metadata.column(index).name())) + .collect::>>()? + .join(", "); + + let mut node_children = vec![ + FormatTreeNode::new(format!( + "output columns: [{}]", + format_output_columns(self.output_schema()?, &ctx.metadata, true) + )), + FormatTreeNode::new(format!("hash keys: [{partition_by}]")), + ]; + + if let Some(top_n) = &self.top_n { + node_children.push(FormatTreeNode::new(format!("top: {}", top_n.top))); + } + + if let Some(info) = &self.stat_info { + node_children.extend(plan_stats_info_to_format_tree(info)); + } + + node_children.extend(children); + Ok(FormatTreeNode::with_children( + "WindowPartition".to_string(), + node_children, + )) + } + #[recursive::recursive] fn try_find_single_data_source(&self) -> Option<&DataSourcePlan> { self.input.try_find_single_data_source() } - fn derive_with( - &self, - handle: &mut Box, - ) -> Box { - let derive_input = self.input.derive_with(handle); - - match handle.derive(self, vec![derive_input]) { - Ok(v) => v, - Err(children) => { - let mut new_window_partition = self.clone(); - assert_eq!(children.len(), 1); - new_window_partition.input = children[0]; - Box::new(new_window_partition) - } - } + fn derive(&self, mut children: Vec>) -> Box { + let mut new_physical_plan = self.clone(); + assert_eq!(children.len(), 1); + new_physical_plan.input = children.pop().unwrap(); + Box::new(new_physical_plan) } } diff --git a/src/query/sql/src/planner/execution/query_executor.rs b/src/query/sql/src/planner/execution/query_executor.rs index b6b6e430f5d35..f8697c4b4fc2f 100644 --- a/src/query/sql/src/planner/execution/query_executor.rs +++ b/src/query/sql/src/planner/execution/query_executor.rs @@ -16,12 +16,15 @@ use async_trait::async_trait; use databend_common_exception::Result; use databend_common_expression::DataBlock; +use crate::executor::IPhysicalPlan; use crate::executor::PhysicalPlan; #[async_trait] pub trait QueryExecutor: Send + Sync { - async fn execute_query_with_physical_plan(&self, plan: &PhysicalPlan) - -> Result>; + async fn execute_query_with_physical_plan( + &self, + plan: &Box, + ) -> Result>; async fn execute_query_with_sql_string(&self, sql: &str) -> Result>; } From 397bca7e529a8c7ef4e37e0bdb6dfa8752cbf8fe Mon Sep 17 00:00:00 2001 From: zhang2014 Date: Thu, 3 Jul 2025 13:38:54 +0800 Subject: [PATCH 7/8] refactor(query): use trait to refactor physical plan --- .../src/interpreters/interpreter_explain.rs | 11 - .../interpreters/interpreter_index_refresh.rs | 2 +- .../src/schedulers/fragments/fragmenter.rs | 2 +- .../src/schedulers/fragments/plan_fragment.rs | 17 +- src/query/sql/src/executor/mod.rs | 1 + src/query/sql/src/executor/physical_plan.rs | 56 +++- .../physical_add_stream_column.rs | 4 + .../physical_aggregate_expand.rs | 5 + .../physical_aggregate_final.rs | 263 +++++++++--------- .../physical_aggregate_partial.rs | 4 + .../physical_plans/physical_async_func.rs | 5 + .../physical_plans/physical_broadcast.rs | 8 + .../physical_plans/physical_cache_scan.rs | 5 + .../physical_column_mutation.rs | 4 + .../physical_plans/physical_commit_sink.rs | 4 + .../physical_plans/physical_compact_source.rs | 4 + .../physical_constant_table_scan.rs | 5 + .../physical_copy_into_location.rs | 5 + .../physical_copy_into_table.rs | 5 + .../physical_distributed_insert_select.rs | 5 + .../physical_plans/physical_eval_scalar.rs | 4 + .../physical_plans/physical_exchange.rs | 5 + .../physical_plans/physical_exchange_sink.rs | 5 + .../physical_exchange_source.rs | 5 + .../physical_expression_scan.rs | 5 + .../physical_plans/physical_filter.rs | 4 + .../physical_plans/physical_hash_join.rs | 87 +++--- .../executor/physical_plans/physical_limit.rs | 4 + .../physical_multi_table_insert.rs | 28 ++ .../physical_plans/physical_mutation.rs | 4 + .../physical_mutation_into_organize.rs | 5 + .../physical_mutation_into_split.rs | 5 + .../physical_mutation_manipulate.rs | 4 + .../physical_mutation_source.rs | 5 + .../physical_plans/physical_project_set.rs | 5 + .../physical_plans/physical_r_cte_scan.rs | 4 + .../physical_plans/physical_range_join.rs | 4 + .../physical_plans/physical_recluster.rs | 8 + .../physical_replace_async_source.rs | 5 + .../physical_replace_deduplicate.rs | 4 + .../physical_plans/physical_replace_into.rs | 5 + .../physical_plans/physical_row_fetch.rs | 5 + .../executor/physical_plans/physical_sort.rs | 5 + .../physical_plans/physical_table_scan.rs | 4 + .../executor/physical_plans/physical_udf.rs | 4 + .../physical_plans/physical_union_all.rs | 5 + .../physical_plans/physical_window.rs | 4 + .../physical_window_partition.rs | 5 + 48 files changed, 449 insertions(+), 203 deletions(-) diff --git a/src/query/service/src/interpreters/interpreter_explain.rs b/src/query/service/src/interpreters/interpreter_explain.rs index 708f0388ed71c..6a15a1046bc57 100644 --- a/src/query/service/src/interpreters/interpreter_explain.rs +++ b/src/query/service/src/interpreters/interpreter_explain.rs @@ -359,17 +359,6 @@ impl ExplainInterpreter { Ok(vec![DataBlock::new_from_columns(vec![formatted_plan])]) } - pub fn explain_join_order( - &self, - plan: &PhysicalPlan, - metadata: &MetadataRef, - ) -> Result> { - let result = plan.format_join(metadata)?.format_pretty()?; - let line_split_result: Vec<&str> = result.lines().collect(); - let formatted_plan = StringType::from_data(line_split_result); - Ok(vec![DataBlock::new_from_columns(vec![formatted_plan])]) - } - fn format_pipeline(build_res: &PipelineBuildResult) -> Vec { let mut blocks = Vec::with_capacity(1 + build_res.sources_pipelines.len()); // Format root pipeline diff --git a/src/query/service/src/interpreters/interpreter_index_refresh.rs b/src/query/service/src/interpreters/interpreter_index_refresh.rs index 341150b8066e8..7b687d38a385e 100644 --- a/src/query/service/src/interpreters/interpreter_index_refresh.rs +++ b/src/query/service/src/interpreters/interpreter_index_refresh.rs @@ -388,7 +388,7 @@ impl DeriveHandle for ReadSourceDeriveHandle { v: &Box, children: Vec>, ) -> std::result::Result, Vec>> { - let Some(table_scan) = v.down_cast::() else { + let Some(table_scan) = v.downcast_ref::() else { return Err(children); }; diff --git a/src/query/service/src/schedulers/fragments/fragmenter.rs b/src/query/service/src/schedulers/fragments/fragmenter.rs index 3446e35a00193..dbc9872517533 100644 --- a/src/query/service/src/schedulers/fragments/fragmenter.rs +++ b/src/query/service/src/schedulers/fragments/fragmenter.rs @@ -161,7 +161,7 @@ impl DeriveHandle for Fragmenter { v: &Box, children: Vec>, ) -> std::result::Result, Vec>> { - let Some(compact_source) = v.down_cast::() else { + let Some(compact_source) = v.downcast_ref::() else { return Err(children); }; } diff --git a/src/query/service/src/schedulers/fragments/plan_fragment.rs b/src/query/service/src/schedulers/fragments/plan_fragment.rs index b36ac62a9b495..79480c6dcb1d5 100644 --- a/src/query/service/src/schedulers/fragments/plan_fragment.rs +++ b/src/query/service/src/schedulers/fragments/plan_fragment.rs @@ -36,6 +36,7 @@ use databend_common_sql::executor::physical_plans::ReplaceInto; use databend_common_sql::executor::physical_plans::TableScan; use databend_common_sql::executor::DeriveHandle; use databend_common_sql::executor::IPhysicalPlan; +use databend_common_sql::executor::PhysicalPlanDynExt; use databend_common_storages_fuse::TableContext; use databend_storages_common_table_meta::meta::BlockSlotDescription; use databend_storages_common_table_meta::meta::Location; @@ -228,7 +229,7 @@ impl PlanFragment { ctx: Arc, fragment_actions: &mut QueryFragmentActions, ) -> Result<()> { - let Some(plan) = self.plan.down_cast::() else { + let Some(plan) = self.plan.downcast_ref::() else { unreachable!("logic error"); }; @@ -490,7 +491,7 @@ impl DeriveHandle for ReadSourceDeriveHandle { v: &Box, children: Vec>, ) -> std::result::Result, Vec>> { - if let Some(table_scan) = v.down_cast::() { + if let Some(table_scan) = v.downcast_ref::() { let source = self.sources.remove(&table_scan.get_id()).ok_or_else(|| { ErrorCode::Internal(format!( "Cannot find data source for table scan plan {}", @@ -502,7 +503,7 @@ impl DeriveHandle for ReadSourceDeriveHandle { source: Box::new(DataSourcePlan::try_from(source)?), ..table_scan.clone() })); - } else if let Some(table_scan) = v.down_cast::() { + } else if let Some(table_scan) = v.downcast_ref::() { let source = self.sources.remove(&table_scan.get_id()).ok_or_else(|| { ErrorCode::Internal(format!( "Cannot find data source for constant table scan plan {}", @@ -539,7 +540,7 @@ impl DeriveHandle for ReclusterDeriveHandle { v: &Box, children: Vec>, ) -> std::result::Result, Vec>> { - let Some(recluster) = v.down_cast::() else { + let Some(recluster) = v.downcast_ref::() else { return Err(children); }; @@ -566,7 +567,7 @@ impl DeriveHandle for MutationSourceDeriveHandle { v: &Box, children: Vec>, ) -> std::result::Result, Vec>> { - let Some(mutation_source) = v.down_cast::() else { + let Some(mutation_source) = v.downcast_ref::() else { return Err(children); }; @@ -593,7 +594,7 @@ impl DeriveHandle for CompactSourceDeriveHandle { v: &Box, children: Vec>, ) -> std::result::Result, Vec>> { - let Some(compact_source) = v.down_cast::() else { + let Some(compact_source) = v.downcast_ref::() else { return Err(children); }; @@ -631,7 +632,7 @@ impl DeriveHandle for ReplaceDeriveHandle { v: &Box, children: Vec>, ) -> std::result::Result, Vec>> { - if let Some(replace_into) = v.down_cast::() { + if let Some(replace_into) = v.downcast_ref::() { assert_eq!(children.len(), 1); return Ok(Box::new(ReplaceInto { input: children[0], @@ -640,7 +641,7 @@ impl DeriveHandle for ReplaceDeriveHandle { block_slots: self.slot.clone(), ..replace_into.clone() })); - } else if let Some(replace_deduplicate) = v.down_cast::() { + } else if let Some(replace_deduplicate) = v.downcast_ref::() { assert_eq!(children.len(), 1); return Ok(Box::new(ReplaceDeduplicate { input: children[0], diff --git a/src/query/sql/src/executor/mod.rs b/src/query/sql/src/executor/mod.rs index a84af92bfd606..3782dee62da8d 100644 --- a/src/query/sql/src/executor/mod.rs +++ b/src/query/sql/src/executor/mod.rs @@ -25,6 +25,7 @@ pub mod table_read_plan; pub use physical_plan::DeriveHandle; pub use physical_plan::IPhysicalPlan; pub use physical_plan::PhysicalPlan; +pub use physical_plan::PhysicalPlanDynExt; pub use physical_plan::PhysicalPlanMeta; pub use physical_plan_builder::MutationBuildInfo; pub use physical_plan_builder::PhysicalPlanBuilder; diff --git a/src/query/sql/src/executor/physical_plan.rs b/src/query/sql/src/executor/physical_plan.rs index 2de83edceef74..4dc78e202dc32 100644 --- a/src/query/sql/src/executor/physical_plan.rs +++ b/src/query/sql/src/executor/physical_plan.rs @@ -13,6 +13,7 @@ // limitations under the License. use std::any::Any; +use std::any::TypeId; use std::collections::HashMap; use std::fmt::Debug; @@ -110,7 +111,9 @@ pub trait DeriveHandle { } #[typetag::serde] -pub trait IPhysicalPlan: Debug { +pub trait IPhysicalPlan: Debug + Send + Sync + 'static { + fn as_any(&self) -> &dyn Any; + fn get_meta(&self) -> &PhysicalPlanMeta; fn get_meta_mut(&mut self) -> &mut PhysicalPlanMeta; @@ -214,20 +217,12 @@ pub trait IPhysicalPlan: Debug { pub trait PhysicalPlanExt { fn clone_box(&self) -> Box; - - fn down_cast(&self) -> Option<&T>; } -impl PhysicalPlanExt for T -where T: 'static + IPhysicalPlan + Clone -{ +impl PhysicalPlanExt for T { fn clone_box(&self) -> Box { Box::new(self.clone()) } - - fn down_cast(&self) -> Option<&T> { - todo!() - } } pub trait PhysicalPlanDynExt { @@ -249,9 +244,15 @@ pub trait PhysicalPlanDynExt { profs: &HashMap, ctx: &mut FormatContext<'_>, ) -> Result>; + + fn downcast_ref(&self) -> Option<&To>; + + fn downcast_mut_ref(&mut self) -> Option<&mut To>; + + fn derive_with(&self, handle: &mut Box) -> Box; } -impl PhysicalPlanDynExt for Box { +impl PhysicalPlanDynExt for Box { fn to_format_tree( &self, profs: &HashMap, @@ -286,13 +287,42 @@ impl PhysicalPlanDynExt for Box { Ok(format_tree_node) } + + fn downcast_ref(&self) -> Option<&To> { + self.as_any().downcast_ref() + } + + fn downcast_mut_ref(&mut self) -> Option<&mut To> { + // self.as_any().downcast_mut() + unimplemented!() + } + + fn derive_with(&self, handle: &mut Box) -> Box { + let mut children = vec![]; + for child in self.children() { + children.push(child.derive_with(handle)); + } + + self.derive(children) + } } -unsafe impl Send for dyn IPhysicalPlan {} +struct DummyDeriveHandle; + +impl DeriveHandle for DummyDeriveHandle { + fn derive( + &mut self, + _: &Box, + children: Vec>, + ) -> std::result::Result, Vec>> { + Err(children) + } +} impl Clone for Box { fn clone(&self) -> Self { - self.clone_box() + let mut handle: Box = Box::new(DummyDeriveHandle); + self.derive_with(&mut handle) } } diff --git a/src/query/sql/src/executor/physical_plans/physical_add_stream_column.rs b/src/query/sql/src/executor/physical_plans/physical_add_stream_column.rs index 1fb5617730eed..c0431407871f3 100644 --- a/src/query/sql/src/executor/physical_plans/physical_add_stream_column.rs +++ b/src/query/sql/src/executor/physical_plans/physical_add_stream_column.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::any::Any; use std::collections::HashMap; use databend_common_ast::ast::FormatTreeNode; @@ -51,6 +52,9 @@ pub struct AddStreamColumn { #[typetag::serde] impl IPhysicalPlan for AddStreamColumn { + fn as_any(&self) -> &dyn Any { + self + } fn get_meta(&self) -> &PhysicalPlanMeta { &self.meta } diff --git a/src/query/sql/src/executor/physical_plans/physical_aggregate_expand.rs b/src/query/sql/src/executor/physical_plans/physical_aggregate_expand.rs index a0713263c35d5..d2a00e9b3ca35 100644 --- a/src/query/sql/src/executor/physical_plans/physical_aggregate_expand.rs +++ b/src/query/sql/src/executor/physical_plans/physical_aggregate_expand.rs @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::any::Any; + use databend_common_ast::ast::FormatTreeNode; use databend_common_catalog::plan::DataSourcePlan; use databend_common_exception::Result; @@ -46,6 +48,9 @@ pub struct AggregateExpand { #[typetag::serde] impl IPhysicalPlan for AggregateExpand { + fn as_any(&self) -> &dyn Any { + self + } fn get_meta(&self) -> &PhysicalPlanMeta { &self.meta } diff --git a/src/query/sql/src/executor/physical_plans/physical_aggregate_final.rs b/src/query/sql/src/executor/physical_plans/physical_aggregate_final.rs index 7c7e7c539c229..131001c857292 100644 --- a/src/query/sql/src/executor/physical_plans/physical_aggregate_final.rs +++ b/src/query/sql/src/executor/physical_plans/physical_aggregate_final.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::any::Any; use std::collections::HashMap; use std::sync::Arc; @@ -32,6 +33,7 @@ use crate::executor::format::plan_stats_info_to_format_tree; use crate::executor::format::pretty_display_agg_desc; use crate::executor::format::FormatContext; use crate::executor::physical_plan::DeriveHandle; +use crate::executor::physical_plan::PhysicalPlanDynExt; use crate::executor::physical_plans::AggregateExpand; use crate::executor::physical_plans::AggregateFunctionDesc; use crate::executor::physical_plans::AggregateFunctionSignature; @@ -63,6 +65,9 @@ pub struct AggregateFinal { #[typetag::serde] impl IPhysicalPlan for AggregateFinal { + fn as_any(&self) -> &dyn Any { + self + } fn get_meta(&self) -> &PhysicalPlanMeta { &self.meta } @@ -360,117 +365,116 @@ impl PhysicalPlanBuilder { (desc, limit) }); - match input { - PhysicalPlan::Exchange(Exchange { input, kind, .. }) - if group_by_shuffle_mode == "before_merge" => - { - let aggregate_partial = if let Some(grouping_sets) = agg.grouping_sets { - let expand = Box::new(AggregateExpand { - input, - grouping_sets, - group_bys: group_items.clone(), - stat_info: Some(stat_info.clone()), - meta: PhysicalPlanMeta::new("AggregateExpand"), - }); - - Box::new(AggregatePartial { - input: expand, - agg_funcs, - enable_experimental_aggregate_hashtable, - group_by_display, - group_by: group_items, - stat_info: Some(stat_info), - rank_limit: None, - meta: PhysicalPlanMeta::new("AggregatePartial"), - }) - } else { - Box::new(AggregatePartial { - input, - agg_funcs, - rank_limit, - group_by_display, - enable_experimental_aggregate_hashtable, - group_by: group_items, - stat_info: Some(stat_info), - meta: PhysicalPlanMeta::new("AggregatePartial"), + if group_by_shuffle_mode == "before_merge" + && let Some(exchange) = input.downcast_ref::() + { + let kind = exchange.kind.clone(); + let aggregate_partial = if let Some(grouping_sets) = agg.grouping_sets { + let expand = Box::new(AggregateExpand { + grouping_sets, + input: exchange.input.clone(), + group_bys: group_items.clone(), + stat_info: Some(stat_info.clone()), + meta: PhysicalPlanMeta::new("AggregateExpand"), + }); + + Box::new(AggregatePartial { + input: expand, + agg_funcs, + enable_experimental_aggregate_hashtable, + group_by_display, + group_by: group_items, + stat_info: Some(stat_info), + rank_limit: None, + meta: PhysicalPlanMeta::new("AggregatePartial"), + }) + } else { + Box::new(AggregatePartial { + input, + agg_funcs, + rank_limit, + group_by_display, + enable_experimental_aggregate_hashtable, + group_by: group_items, + stat_info: Some(stat_info), + meta: PhysicalPlanMeta::new("AggregatePartial"), + }) + }; + + let keys = { + let schema = aggregate_partial.output_schema()?; + let end = schema.num_fields(); + let start = end - aggregate_partial.group_by.len(); + (start..end) + .map(|id| RemoteExpr::ColumnRef { + span: None, + id, + data_type: schema.field(id).data_type().clone(), + display_name: (id - start).to_string(), }) + .collect() + }; + + Box::new(Exchange { + keys, + kind, + ignore_exchange: false, + allow_adjust_parallelism: true, + meta: PhysicalPlanMeta::new("Exchange"), + input: aggregate_partial, + }) + } else { + if let Some(grouping_sets) = agg.grouping_sets { + let expand = AggregateExpand { + input, + grouping_sets, + group_bys: group_items.clone(), + stat_info: Some(stat_info.clone()), + meta: PhysicalPlanMeta::new("AggregateExpand"), }; - let keys = { - let schema = aggregate_partial.output_schema()?; - let end = schema.num_fields(); - let start = end - aggregate_partial.group_by.len(); - (start..end) - .map(|id| RemoteExpr::ColumnRef { - span: None, - id, - data_type: schema.field(id).data_type().clone(), - display_name: (id - start).to_string(), - }) - .collect() - }; - - Box::new(Exchange { - kind, - keys, - ignore_exchange: false, - allow_adjust_parallelism: true, - meta: PhysicalPlanMeta::new("Exchange"), - input: aggregate_partial, + Box::new(AggregatePartial { + agg_funcs, + group_by_display, + enable_experimental_aggregate_hashtable, + rank_limit: None, + group_by: group_items, + input: Box::new(expand), + stat_info: Some(stat_info), + meta: PhysicalPlanMeta::new("AggregatePartial"), + }) + } else { + Box::new(AggregatePartial { + input, + agg_funcs, + enable_experimental_aggregate_hashtable, + group_by_display, + group_by: group_items, + stat_info: Some(stat_info), + rank_limit, + meta: PhysicalPlanMeta::new("AggregatePartial"), }) - } - _ => { - if let Some(grouping_sets) = agg.grouping_sets { - let expand = AggregateExpand { - input, - grouping_sets, - group_bys: group_items.clone(), - stat_info: Some(stat_info.clone()), - meta: PhysicalPlanMeta::new("AggregateExpand"), - }; - - Box::new(AggregatePartial { - agg_funcs, - group_by_display, - enable_experimental_aggregate_hashtable, - rank_limit: None, - group_by: group_items, - input: Box::new(expand), - stat_info: Some(stat_info), - meta: PhysicalPlanMeta::new("AggregatePartial"), - }) - } else { - Box::new(AggregatePartial { - input, - agg_funcs, - enable_experimental_aggregate_hashtable, - group_by_display, - group_by: group_items, - stat_info: Some(stat_info), - rank_limit, - meta: PhysicalPlanMeta::new("AggregatePartial"), - }) - } } } } // Hack to get before group by schema, we should refactor this AggregateMode::Final => { - let input_schema = match input { - PhysicalPlan::AggregatePartial(ref agg) => agg.input.output_schema()?, + let input_schema = { + let mut plan = &input; - PhysicalPlan::Exchange(Exchange { - input: box PhysicalPlan::AggregatePartial(ref agg), - .. - }) => agg.input.output_schema()?, + if let Some(exchange) = plan.downcast_ref::() { + plan = &exchange.input; + } - _ => { + let Some(aggregate) = plan.downcast_ref::() else { return Err(ErrorCode::Internal(format!( "invalid input physical plan: {}", input.get_name(), ))); - } + }; + + aggregate.input.output_schema()? }; let mut agg_funcs: Vec = agg @@ -592,45 +596,46 @@ impl PhysicalPlanBuilder { } } - match input { - PhysicalPlan::AggregatePartial(ref partial) => { - let group_by_display = partial.group_by_display.clone(); - let before_group_by_schema = partial.input.output_schema()?; - - Box::new(AggregateFinal { - input, - agg_funcs, - group_by_display, - before_group_by_schema, - group_by: group_items, - stat_info: Some(stat_info), - meta: PhysicalPlanMeta::new("AggregateFinal"), - }) - } - - PhysicalPlan::Exchange(Exchange { - input: box PhysicalPlan::AggregatePartial(ref partial), - .. - }) => { - let before_group_by_schema = partial.input.output_schema()?; - - Box::new(AggregateFinal { - input, - agg_funcs, - before_group_by_schema, - group_by: group_items, - stat_info: Some(stat_info), - group_by_display: partial.group_by_display.clone(), - meta: PhysicalPlanMeta::new("AggregateFinal"), - }) - } + if let Some(partial) = input.downcast_ref::() { + let group_by_display = partial.group_by_display.clone(); + let before_group_by_schema = partial.input.output_schema()?; + + Box::new(AggregateFinal { + input, + agg_funcs, + group_by_display, + before_group_by_schema, + group_by: group_items, + stat_info: Some(stat_info), + meta: PhysicalPlanMeta::new("AggregateFinal"), + }) + } else { + let Some(exchange) = input.downcast_ref::() else { + return Err(ErrorCode::Internal(format!( + "invalid input physical plan: {}", + input.get_name(), + ))); + }; - _ => { + let Some(partial) = exchange.input.downcast_ref::() else { return Err(ErrorCode::Internal(format!( "invalid input physical plan: {}", input.get_name(), ))); - } + }; + + let group_by_display = partial.group_by_display.clone(); + let before_group_by_schema = partial.input.output_schema()?; + + Box::new(AggregateFinal { + input, + agg_funcs, + group_by_display, + before_group_by_schema, + group_by: group_items, + stat_info: Some(stat_info), + meta: PhysicalPlanMeta::new("AggregateFinal"), + }) } } AggregateMode::Initial => { diff --git a/src/query/sql/src/executor/physical_plans/physical_aggregate_partial.rs b/src/query/sql/src/executor/physical_plans/physical_aggregate_partial.rs index 636b113bcb0e6..c13d758c1fad5 100644 --- a/src/query/sql/src/executor/physical_plans/physical_aggregate_partial.rs +++ b/src/query/sql/src/executor/physical_plans/physical_aggregate_partial.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::any::Any; use std::collections::HashMap; use databend_common_ast::ast::FormatTreeNode; @@ -54,6 +55,9 @@ pub struct AggregatePartial { #[typetag::serde] impl IPhysicalPlan for AggregatePartial { + fn as_any(&self) -> &dyn Any { + self + } fn get_meta(&self) -> &PhysicalPlanMeta { &self.meta } diff --git a/src/query/sql/src/executor/physical_plans/physical_async_func.rs b/src/query/sql/src/executor/physical_plans/physical_async_func.rs index a52d84bbeb02e..a2500291cfb89 100644 --- a/src/query/sql/src/executor/physical_plans/physical_async_func.rs +++ b/src/query/sql/src/executor/physical_plans/physical_async_func.rs @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::any::Any; + use databend_common_ast::ast::FormatTreeNode; use databend_common_catalog::plan::DataSourcePlan; use databend_common_exception::ErrorCode; @@ -48,6 +50,9 @@ pub struct AsyncFunction { #[typetag::serde] impl IPhysicalPlan for AsyncFunction { + fn as_any(&self) -> &dyn Any { + self + } fn get_meta(&self) -> &PhysicalPlanMeta { &self.meta } diff --git a/src/query/sql/src/executor/physical_plans/physical_broadcast.rs b/src/query/sql/src/executor/physical_plans/physical_broadcast.rs index 2cea4c451a21a..29337e830da09 100644 --- a/src/query/sql/src/executor/physical_plans/physical_broadcast.rs +++ b/src/query/sql/src/executor/physical_plans/physical_broadcast.rs @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::any::Any; + use databend_common_ast::ast::FormatTreeNode; use databend_common_catalog::table_context::TableContext; use databend_common_exception::Result; @@ -33,6 +35,9 @@ pub struct BroadcastSource { #[typetag::serde] impl IPhysicalPlan for BroadcastSource { + fn as_any(&self) -> &dyn Any { + self + } fn get_meta(&self) -> &PhysicalPlanMeta { &self.meta } @@ -56,6 +61,9 @@ pub struct BroadcastSink { #[typetag::serde] impl IPhysicalPlan for BroadcastSink { + fn as_any(&self) -> &dyn Any { + self + } fn get_meta(&self) -> &PhysicalPlanMeta { &self.meta } diff --git a/src/query/sql/src/executor/physical_plans/physical_cache_scan.rs b/src/query/sql/src/executor/physical_plans/physical_cache_scan.rs index ac464d95c4984..fee02bc52e175 100644 --- a/src/query/sql/src/executor/physical_plans/physical_cache_scan.rs +++ b/src/query/sql/src/executor/physical_plans/physical_cache_scan.rs @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::any::Any; + use databend_common_ast::ast::FormatTreeNode; use databend_common_exception::Result; use databend_common_expression::DataSchemaRef; @@ -36,6 +38,9 @@ pub struct CacheScan { #[typetag::serde] impl IPhysicalPlan for CacheScan { + fn as_any(&self) -> &dyn Any { + self + } fn get_meta(&self) -> &PhysicalPlanMeta { &self.meta } diff --git a/src/query/sql/src/executor/physical_plans/physical_column_mutation.rs b/src/query/sql/src/executor/physical_plans/physical_column_mutation.rs index 8648b6f471473..6ebfad86d8e0c 100644 --- a/src/query/sql/src/executor/physical_plans/physical_column_mutation.rs +++ b/src/query/sql/src/executor/physical_plans/physical_column_mutation.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::any::Any; use std::collections::HashMap; use databend_common_ast::ast::FormatTreeNode; @@ -45,6 +46,9 @@ pub struct ColumnMutation { #[typetag::serde] impl IPhysicalPlan for ColumnMutation { + fn as_any(&self) -> &dyn Any { + self + } fn get_meta(&self) -> &PhysicalPlanMeta { &self.meta } diff --git a/src/query/sql/src/executor/physical_plans/physical_commit_sink.rs b/src/query/sql/src/executor/physical_plans/physical_commit_sink.rs index 339ffffc0bf21..a06d7fb2d62e4 100644 --- a/src/query/sql/src/executor/physical_plans/physical_commit_sink.rs +++ b/src/query/sql/src/executor/physical_plans/physical_commit_sink.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::any::Any; use std::sync::Arc; use databend_common_catalog::plan::DataSourcePlan; @@ -49,6 +50,9 @@ pub struct CommitSink { #[typetag::serde] impl IPhysicalPlan for CommitSink { + fn as_any(&self) -> &dyn Any { + self + } fn get_meta(&self) -> &PhysicalPlanMeta { &self.meta } diff --git a/src/query/sql/src/executor/physical_plans/physical_compact_source.rs b/src/query/sql/src/executor/physical_plans/physical_compact_source.rs index daa11e1e3a2d3..587fd227d3e64 100644 --- a/src/query/sql/src/executor/physical_plans/physical_compact_source.rs +++ b/src/query/sql/src/executor/physical_plans/physical_compact_source.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::any::Any; use std::collections::HashSet; use databend_common_catalog::plan::DataSourcePlan; @@ -47,6 +48,9 @@ pub struct CompactSource { #[typetag::serde] impl IPhysicalPlan for CompactSource { + fn as_any(&self) -> &dyn Any { + self + } fn get_meta(&self) -> &PhysicalPlanMeta { &self.meta } diff --git a/src/query/sql/src/executor/physical_plans/physical_constant_table_scan.rs b/src/query/sql/src/executor/physical_plans/physical_constant_table_scan.rs index ccd6194661dec..ccf4c0182d418 100644 --- a/src/query/sql/src/executor/physical_plans/physical_constant_table_scan.rs +++ b/src/query/sql/src/executor/physical_plans/physical_constant_table_scan.rs @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::any::Any; + use databend_common_ast::ast::FormatTreeNode; use databend_common_exception::Result; use databend_common_expression::Column; @@ -38,6 +40,9 @@ pub struct ConstantTableScan { #[typetag::serde] impl IPhysicalPlan for ConstantTableScan { + fn as_any(&self) -> &dyn Any { + self + } fn get_meta(&self) -> &PhysicalPlanMeta { &self.meta } diff --git a/src/query/sql/src/executor/physical_plans/physical_copy_into_location.rs b/src/query/sql/src/executor/physical_plans/physical_copy_into_location.rs index cb10eecdcb7d0..0d742d09242fd 100644 --- a/src/query/sql/src/executor/physical_plans/physical_copy_into_location.rs +++ b/src/query/sql/src/executor/physical_plans/physical_copy_into_location.rs @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::any::Any; + use databend_common_ast::ast::FormatTreeNode; use databend_common_catalog::plan::DataSourcePlan; use databend_common_exception::Result; @@ -42,6 +44,9 @@ pub struct CopyIntoLocation { #[typetag::serde] impl IPhysicalPlan for CopyIntoLocation { + fn as_any(&self) -> &dyn Any { + self + } fn get_meta(&self) -> &PhysicalPlanMeta { &self.meta } diff --git a/src/query/sql/src/executor/physical_plans/physical_copy_into_table.rs b/src/query/sql/src/executor/physical_plans/physical_copy_into_table.rs index e5f56d1f3211a..4099cf1286435 100644 --- a/src/query/sql/src/executor/physical_plans/physical_copy_into_table.rs +++ b/src/query/sql/src/executor/physical_plans/physical_copy_into_table.rs @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::any::Any; + use databend_common_ast::ast::FormatTreeNode; use databend_common_catalog::plan::DataSourcePlan; use databend_common_catalog::plan::StageTableInfo; @@ -51,6 +53,9 @@ pub struct CopyIntoTable { #[typetag::serde] impl IPhysicalPlan for CopyIntoTable { + fn as_any(&self) -> &dyn Any { + self + } fn get_meta(&self) -> &PhysicalPlanMeta { &self.meta } diff --git a/src/query/sql/src/executor/physical_plans/physical_distributed_insert_select.rs b/src/query/sql/src/executor/physical_plans/physical_distributed_insert_select.rs index 2a9373b08a2c0..4c24da61b2262 100644 --- a/src/query/sql/src/executor/physical_plans/physical_distributed_insert_select.rs +++ b/src/query/sql/src/executor/physical_plans/physical_distributed_insert_select.rs @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::any::Any; + use databend_common_catalog::plan::DataSourcePlan; use databend_common_exception::Result; use databend_common_expression::DataSchemaRef; @@ -38,6 +40,9 @@ pub struct DistributedInsertSelect { #[typetag::serde] impl IPhysicalPlan for DistributedInsertSelect { + fn as_any(&self) -> &dyn Any { + self + } fn get_meta(&self) -> &PhysicalPlanMeta { &self.meta } diff --git a/src/query/sql/src/executor/physical_plans/physical_eval_scalar.rs b/src/query/sql/src/executor/physical_plans/physical_eval_scalar.rs index b23fe5ed6a1ce..68412e0ef12bb 100644 --- a/src/query/sql/src/executor/physical_plans/physical_eval_scalar.rs +++ b/src/query/sql/src/executor/physical_plans/physical_eval_scalar.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::any::Any; use std::collections::BTreeSet; use std::collections::HashMap; use std::collections::HashSet; @@ -65,6 +66,9 @@ pub struct EvalScalar { #[typetag::serde] impl IPhysicalPlan for EvalScalar { + fn as_any(&self) -> &dyn Any { + self + } fn get_meta(&self) -> &PhysicalPlanMeta { &self.meta } diff --git a/src/query/sql/src/executor/physical_plans/physical_exchange.rs b/src/query/sql/src/executor/physical_plans/physical_exchange.rs index 36dcaaaba7361..1a6fc65046cb6 100644 --- a/src/query/sql/src/executor/physical_plans/physical_exchange.rs +++ b/src/query/sql/src/executor/physical_plans/physical_exchange.rs @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::any::Any; + use databend_common_ast::ast::FormatTreeNode; use databend_common_catalog::plan::DataSourcePlan; use databend_common_exception::Result; @@ -44,6 +46,9 @@ pub struct Exchange { #[typetag::serde] impl IPhysicalPlan for Exchange { + fn as_any(&self) -> &dyn Any { + self + } fn get_meta(&self) -> &PhysicalPlanMeta { &self.meta } diff --git a/src/query/sql/src/executor/physical_plans/physical_exchange_sink.rs b/src/query/sql/src/executor/physical_plans/physical_exchange_sink.rs index bef180b672a1c..61837c2875c07 100644 --- a/src/query/sql/src/executor/physical_plans/physical_exchange_sink.rs +++ b/src/query/sql/src/executor/physical_plans/physical_exchange_sink.rs @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::any::Any; + use databend_common_ast::ast::FormatTreeNode; use databend_common_catalog::plan::DataSourcePlan; use databend_common_exception::Result; @@ -46,6 +48,9 @@ pub struct ExchangeSink { #[typetag::serde] impl IPhysicalPlan for ExchangeSink { + fn as_any(&self) -> &dyn Any { + self + } fn get_meta(&self) -> &PhysicalPlanMeta { &self.meta } diff --git a/src/query/sql/src/executor/physical_plans/physical_exchange_source.rs b/src/query/sql/src/executor/physical_plans/physical_exchange_source.rs index 7e34b38ef378a..c0d61ffb56d47 100644 --- a/src/query/sql/src/executor/physical_plans/physical_exchange_source.rs +++ b/src/query/sql/src/executor/physical_plans/physical_exchange_source.rs @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::any::Any; + use databend_common_ast::ast::FormatTreeNode; use databend_common_catalog::plan::DataSourcePlan; use databend_common_exception::Result; @@ -37,6 +39,9 @@ pub struct ExchangeSource { #[typetag::serde] impl IPhysicalPlan for ExchangeSource { + fn as_any(&self) -> &dyn Any { + self + } fn get_meta(&self) -> &PhysicalPlanMeta { &self.meta } diff --git a/src/query/sql/src/executor/physical_plans/physical_expression_scan.rs b/src/query/sql/src/executor/physical_plans/physical_expression_scan.rs index 0a642f82559fe..2a97a750109a2 100644 --- a/src/query/sql/src/executor/physical_plans/physical_expression_scan.rs +++ b/src/query/sql/src/executor/physical_plans/physical_expression_scan.rs @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::any::Any; + use databend_common_ast::ast::FormatTreeNode; use databend_common_exception::Result; use databend_common_expression::ConstantFolder; @@ -41,6 +43,9 @@ pub struct ExpressionScan { #[typetag::serde] impl IPhysicalPlan for ExpressionScan { + fn as_any(&self) -> &dyn Any { + self + } fn get_meta(&self) -> &PhysicalPlanMeta { &self.meta } diff --git a/src/query/sql/src/executor/physical_plans/physical_filter.rs b/src/query/sql/src/executor/physical_plans/physical_filter.rs index 90dd2b716caa4..2f0eb2d56444a 100644 --- a/src/query/sql/src/executor/physical_plans/physical_filter.rs +++ b/src/query/sql/src/executor/physical_plans/physical_filter.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::any::Any; use std::collections::HashMap; use databend_common_ast::ast::FormatTreeNode; @@ -52,6 +53,9 @@ pub struct Filter { #[typetag::serde] impl IPhysicalPlan for Filter { + fn as_any(&self) -> &dyn Any { + self + } fn get_meta(&self) -> &PhysicalPlanMeta { &self.meta } diff --git a/src/query/sql/src/executor/physical_plans/physical_hash_join.rs b/src/query/sql/src/executor/physical_plans/physical_hash_join.rs index 2fefc2c43a4c0..f206c63d87258 100644 --- a/src/query/sql/src/executor/physical_plans/physical_hash_join.rs +++ b/src/query/sql/src/executor/physical_plans/physical_hash_join.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::any::Any; use std::collections::HashMap; use std::collections::HashSet; @@ -36,6 +37,7 @@ use crate::executor::format::format_output_columns; use crate::executor::format::plan_stats_info_to_format_tree; use crate::executor::format::FormatContext; use crate::executor::physical_plan::DeriveHandle; +use crate::executor::physical_plan::PhysicalPlanDynExt; use crate::executor::physical_plans::Exchange; use crate::executor::IPhysicalPlan; use crate::executor::PhysicalPlan; @@ -117,6 +119,9 @@ pub struct HashJoin { #[typetag::serde] impl IPhysicalPlan for HashJoin { + fn as_any(&self) -> &dyn Any { + self + } fn get_meta(&self) -> &PhysicalPlanMeta { &self.meta } @@ -426,47 +431,49 @@ impl PhysicalPlanBuilder { build_side: &mut Box, ) -> Result<()> { // Unify the data types of the left and right exchange keys - if let ( - PhysicalPlan::Exchange(Exchange { - keys: probe_keys, .. - }), - PhysicalPlan::Exchange(Exchange { - keys: build_keys, .. - }), - ) = (probe_side.as_mut(), build_side.as_mut()) + let Some(probe_exchange) = probe_side.downcast_mut_ref::() else { + return Ok(()); + }; + + let Some(build_exchange) = build_side.downcast_mut_ref::() else { + return Ok(()); + }; + + let cast_rules = &BUILTIN_FUNCTIONS.get_auto_cast_rules("eq"); + for (probe_key, build_key) in probe_exchange + .keys + .iter_mut() + .zip(build_exchange.keys.iter_mut()) { - let cast_rules = &BUILTIN_FUNCTIONS.get_auto_cast_rules("eq"); - for (probe_key, build_key) in probe_keys.iter_mut().zip(build_keys.iter_mut()) { - let probe_expr = probe_key.as_expr(&BUILTIN_FUNCTIONS); - let build_expr = build_key.as_expr(&BUILTIN_FUNCTIONS); - let common_ty = common_super_type( - probe_expr.data_type().clone(), - build_expr.data_type().clone(), - cast_rules, - ) - .ok_or_else(|| { - ErrorCode::IllegalDataType(format!( - "Cannot find common type for probe key {:?} and build key {:?}", - &probe_expr, &build_expr - )) - })?; - *probe_key = check_cast( - probe_expr.span(), - false, - probe_expr, - &common_ty, - &BUILTIN_FUNCTIONS, - )? - .as_remote_expr(); - *build_key = check_cast( - build_expr.span(), - false, - build_expr, - &common_ty, - &BUILTIN_FUNCTIONS, - )? - .as_remote_expr(); - } + let probe_expr = probe_key.as_expr(&BUILTIN_FUNCTIONS); + let build_expr = build_key.as_expr(&BUILTIN_FUNCTIONS); + let common_ty = common_super_type( + probe_expr.data_type().clone(), + build_expr.data_type().clone(), + cast_rules, + ) + .ok_or_else(|| { + ErrorCode::IllegalDataType(format!( + "Cannot find common type for probe key {:?} and build key {:?}", + &probe_expr, &build_expr + )) + })?; + *probe_key = check_cast( + probe_expr.span(), + false, + probe_expr, + &common_ty, + &BUILTIN_FUNCTIONS, + )? + .as_remote_expr(); + *build_key = check_cast( + build_expr.span(), + false, + build_expr, + &common_ty, + &BUILTIN_FUNCTIONS, + )? + .as_remote_expr(); } Ok(()) diff --git a/src/query/sql/src/executor/physical_plans/physical_limit.rs b/src/query/sql/src/executor/physical_plans/physical_limit.rs index 65e8df9a20975..6e576edb457b1 100644 --- a/src/query/sql/src/executor/physical_plans/physical_limit.rs +++ b/src/query/sql/src/executor/physical_plans/physical_limit.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::any::Any; use std::collections::HashMap; use databend_common_ast::ast::FormatTreeNode; @@ -49,6 +50,9 @@ pub struct Limit { #[typetag::serde] impl IPhysicalPlan for Limit { + fn as_any(&self) -> &dyn Any { + self + } fn get_meta(&self) -> &PhysicalPlanMeta { &self.meta } diff --git a/src/query/sql/src/executor/physical_plans/physical_multi_table_insert.rs b/src/query/sql/src/executor/physical_plans/physical_multi_table_insert.rs index ac468ca33e846..ce330fa617077 100644 --- a/src/query/sql/src/executor/physical_plans/physical_multi_table_insert.rs +++ b/src/query/sql/src/executor/physical_plans/physical_multi_table_insert.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::any::Any; use std::sync::Arc; use databend_common_ast::ast::FormatTreeNode; @@ -42,6 +43,9 @@ pub struct Duplicate { #[typetag::serde] impl IPhysicalPlan for Duplicate { + fn as_any(&self) -> &dyn Any { + self + } fn get_meta(&self) -> &PhysicalPlanMeta { &self.meta } @@ -95,6 +99,9 @@ pub struct Shuffle { #[typetag::serde] impl IPhysicalPlan for Shuffle { + fn as_any(&self) -> &dyn Any { + self + } fn get_meta(&self) -> &PhysicalPlanMeta { &self.meta } @@ -167,6 +174,9 @@ pub struct ChunkFilter { #[typetag::serde] impl IPhysicalPlan for ChunkFilter { + fn as_any(&self) -> &dyn Any { + self + } fn get_meta(&self) -> &PhysicalPlanMeta { &self.meta } @@ -231,6 +241,9 @@ pub struct ChunkEvalScalar { #[typetag::serde] impl IPhysicalPlan for ChunkEvalScalar { + fn as_any(&self) -> &dyn Any { + self + } fn get_meta(&self) -> &PhysicalPlanMeta { &self.meta } @@ -306,6 +319,9 @@ pub struct ChunkCastSchema { #[typetag::serde] impl IPhysicalPlan for ChunkCastSchema { + fn as_any(&self) -> &dyn Any { + self + } fn get_meta(&self) -> &PhysicalPlanMeta { &self.meta } @@ -356,6 +372,9 @@ pub struct ChunkFillAndReorder { #[typetag::serde] impl IPhysicalPlan for ChunkFillAndReorder { + fn as_any(&self) -> &dyn Any { + self + } fn get_meta(&self) -> &PhysicalPlanMeta { &self.meta } @@ -407,6 +426,9 @@ pub struct ChunkAppendData { #[typetag::serde] impl IPhysicalPlan for ChunkAppendData { + fn as_any(&self) -> &dyn Any { + self + } fn get_meta(&self) -> &PhysicalPlanMeta { &self.meta } @@ -460,6 +482,9 @@ pub struct ChunkMerge { #[typetag::serde] impl IPhysicalPlan for ChunkMerge { + fn as_any(&self) -> &dyn Any { + self + } fn get_meta(&self) -> &PhysicalPlanMeta { &self.meta } @@ -508,6 +533,9 @@ pub struct ChunkCommitInsert { #[typetag::serde] impl IPhysicalPlan for ChunkCommitInsert { + fn as_any(&self) -> &dyn Any { + self + } fn get_meta(&self) -> &PhysicalPlanMeta { &self.meta } diff --git a/src/query/sql/src/executor/physical_plans/physical_mutation.rs b/src/query/sql/src/executor/physical_plans/physical_mutation.rs index 23f0bd69020ae..af7dc666838a0 100644 --- a/src/query/sql/src/executor/physical_plans/physical_mutation.rs +++ b/src/query/sql/src/executor/physical_plans/physical_mutation.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::any::Any; use std::collections::HashMap; use std::sync::Arc; @@ -103,6 +104,9 @@ pub struct Mutation { #[typetag::serde] impl IPhysicalPlan for Mutation { + fn as_any(&self) -> &dyn Any { + self + } fn get_meta(&self) -> &PhysicalPlanMeta { &self.meta } diff --git a/src/query/sql/src/executor/physical_plans/physical_mutation_into_organize.rs b/src/query/sql/src/executor/physical_plans/physical_mutation_into_organize.rs index 7c761c6f1aca6..7afc6aea326d7 100644 --- a/src/query/sql/src/executor/physical_plans/physical_mutation_into_organize.rs +++ b/src/query/sql/src/executor/physical_plans/physical_mutation_into_organize.rs @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::any::Any; + use databend_common_ast::ast::FormatTreeNode; use databend_common_exception::Result; use databend_common_expression::DataSchemaRef; @@ -32,6 +34,9 @@ pub struct MutationOrganize { #[typetag::serde] impl IPhysicalPlan for MutationOrganize { + fn as_any(&self) -> &dyn Any { + self + } fn get_meta(&self) -> &PhysicalPlanMeta { &self.meta } diff --git a/src/query/sql/src/executor/physical_plans/physical_mutation_into_split.rs b/src/query/sql/src/executor/physical_plans/physical_mutation_into_split.rs index f5ac2c9849c05..e1ec705e8d5a7 100644 --- a/src/query/sql/src/executor/physical_plans/physical_mutation_into_split.rs +++ b/src/query/sql/src/executor/physical_plans/physical_mutation_into_split.rs @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::any::Any; + use databend_common_ast::ast::FormatTreeNode; use databend_common_exception::Result; use databend_common_expression::DataSchemaRef; @@ -32,6 +34,9 @@ pub struct MutationSplit { #[typetag::serde] impl IPhysicalPlan for MutationSplit { + fn as_any(&self) -> &dyn Any { + self + } fn get_meta(&self) -> &PhysicalPlanMeta { &self.meta } diff --git a/src/query/sql/src/executor/physical_plans/physical_mutation_manipulate.rs b/src/query/sql/src/executor/physical_plans/physical_mutation_manipulate.rs index 1cbc7e46fe7de..fafb4ced660a0 100644 --- a/src/query/sql/src/executor/physical_plans/physical_mutation_manipulate.rs +++ b/src/query/sql/src/executor/physical_plans/physical_mutation_manipulate.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::any::Any; use std::collections::HashMap; use databend_common_ast::ast::FormatTreeNode; @@ -53,6 +54,9 @@ pub struct MutationManipulate { #[typetag::serde] impl IPhysicalPlan for MutationManipulate { + fn as_any(&self) -> &dyn Any { + self + } fn get_meta(&self) -> &PhysicalPlanMeta { &self.meta } diff --git a/src/query/sql/src/executor/physical_plans/physical_mutation_source.rs b/src/query/sql/src/executor/physical_plans/physical_mutation_source.rs index 18cde5ce546d4..58d0955355673 100644 --- a/src/query/sql/src/executor/physical_plans/physical_mutation_source.rs +++ b/src/query/sql/src/executor/physical_plans/physical_mutation_source.rs @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::any::Any; + use databend_common_ast::ast::FormatTreeNode; use databend_common_catalog::plan::DataSourcePlan; use databend_common_catalog::plan::Filters; @@ -59,6 +61,9 @@ pub struct MutationSource { #[typetag::serde] impl IPhysicalPlan for MutationSource { + fn as_any(&self) -> &dyn Any { + self + } fn get_meta(&self) -> &PhysicalPlanMeta { &self.meta } diff --git a/src/query/sql/src/executor/physical_plans/physical_project_set.rs b/src/query/sql/src/executor/physical_plans/physical_project_set.rs index a53732106a524..7b4dc84e2ef6b 100644 --- a/src/query/sql/src/executor/physical_plans/physical_project_set.rs +++ b/src/query/sql/src/executor/physical_plans/physical_project_set.rs @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::any::Any; + use databend_common_ast::ast::FormatTreeNode; use databend_common_catalog::plan::DataSourcePlan; use databend_common_exception::Result; @@ -50,6 +52,9 @@ pub struct ProjectSet { #[typetag::serde] impl IPhysicalPlan for ProjectSet { + fn as_any(&self) -> &dyn Any { + self + } fn get_meta(&self) -> &PhysicalPlanMeta { &self.meta } diff --git a/src/query/sql/src/executor/physical_plans/physical_r_cte_scan.rs b/src/query/sql/src/executor/physical_plans/physical_r_cte_scan.rs index 9dd806a500d97..3f92f60abc1bd 100644 --- a/src/query/sql/src/executor/physical_plans/physical_r_cte_scan.rs +++ b/src/query/sql/src/executor/physical_plans/physical_r_cte_scan.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::any::Any; use std::fmt::Display; use databend_common_exception::Result; @@ -35,6 +36,9 @@ pub struct RecursiveCteScan { #[typetag::serde] impl IPhysicalPlan for RecursiveCteScan { + fn as_any(&self) -> &dyn Any { + self + } fn get_meta(&self) -> &PhysicalPlanMeta { &self.meta } diff --git a/src/query/sql/src/executor/physical_plans/physical_range_join.rs b/src/query/sql/src/executor/physical_plans/physical_range_join.rs index ce4620df2d89a..82d596148f141 100644 --- a/src/query/sql/src/executor/physical_plans/physical_range_join.rs +++ b/src/query/sql/src/executor/physical_plans/physical_range_join.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::any::Any; use std::sync::Arc; use databend_common_ast::ast::FormatTreeNode; @@ -65,6 +66,9 @@ pub struct RangeJoin { #[typetag::serde] impl IPhysicalPlan for RangeJoin { + fn as_any(&self) -> &dyn Any { + self + } fn get_meta(&self) -> &PhysicalPlanMeta { &self.meta } diff --git a/src/query/sql/src/executor/physical_plans/physical_recluster.rs b/src/query/sql/src/executor/physical_plans/physical_recluster.rs index d8d46d799f8ff..2b5c6154db42e 100644 --- a/src/query/sql/src/executor/physical_plans/physical_recluster.rs +++ b/src/query/sql/src/executor/physical_plans/physical_recluster.rs @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::any::Any; + use databend_common_catalog::plan::DataSourcePlan; use databend_common_catalog::plan::ReclusterTask; use databend_common_exception::Result; @@ -34,6 +36,9 @@ pub struct Recluster { #[typetag::serde] impl IPhysicalPlan for Recluster { + fn as_any(&self) -> &dyn Any { + self + } fn get_meta(&self) -> &PhysicalPlanMeta { &self.meta } @@ -60,6 +65,9 @@ pub struct HilbertPartition { #[typetag::serde] impl IPhysicalPlan for HilbertPartition { + fn as_any(&self) -> &dyn Any { + self + } fn get_meta(&self) -> &PhysicalPlanMeta { &self.meta } diff --git a/src/query/sql/src/executor/physical_plans/physical_replace_async_source.rs b/src/query/sql/src/executor/physical_plans/physical_replace_async_source.rs index ac3cfda3d6626..ca8003ca3a5e9 100644 --- a/src/query/sql/src/executor/physical_plans/physical_replace_async_source.rs +++ b/src/query/sql/src/executor/physical_plans/physical_replace_async_source.rs @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::any::Any; + use databend_common_catalog::plan::DataSourcePlan; use databend_common_expression::DataSchemaRef; @@ -29,6 +31,9 @@ pub struct ReplaceAsyncSourcer { #[typetag::serde] impl IPhysicalPlan for ReplaceAsyncSourcer { + fn as_any(&self) -> &dyn Any { + self + } fn get_meta(&self) -> &PhysicalPlanMeta { &self.meta } diff --git a/src/query/sql/src/executor/physical_plans/physical_replace_deduplicate.rs b/src/query/sql/src/executor/physical_plans/physical_replace_deduplicate.rs index bb77e5644a59f..9e5d519bb7087 100644 --- a/src/query/sql/src/executor/physical_plans/physical_replace_deduplicate.rs +++ b/src/query/sql/src/executor/physical_plans/physical_replace_deduplicate.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::any::Any; use std::collections::HashMap; use databend_common_catalog::plan::DataSourcePlan; @@ -48,6 +49,9 @@ pub struct ReplaceDeduplicate { #[typetag::serde] impl IPhysicalPlan for ReplaceDeduplicate { + fn as_any(&self) -> &dyn Any { + self + } fn get_meta(&self) -> &PhysicalPlanMeta { &self.meta } diff --git a/src/query/sql/src/executor/physical_plans/physical_replace_into.rs b/src/query/sql/src/executor/physical_plans/physical_replace_into.rs index 1b4ef03fbc146..22b50a53a10ec 100644 --- a/src/query/sql/src/executor/physical_plans/physical_replace_into.rs +++ b/src/query/sql/src/executor/physical_plans/physical_replace_into.rs @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::any::Any; + use databend_common_catalog::plan::DataSourcePlan; use databend_common_exception::Result; use databend_common_expression::BlockThresholds; @@ -44,6 +46,9 @@ pub struct ReplaceInto { #[typetag::serde] impl IPhysicalPlan for ReplaceInto { + fn as_any(&self) -> &dyn Any { + self + } fn get_meta(&self) -> &PhysicalPlanMeta { &self.meta } diff --git a/src/query/sql/src/executor/physical_plans/physical_row_fetch.rs b/src/query/sql/src/executor/physical_plans/physical_row_fetch.rs index 4208f860ce92c..ea7f25309546d 100644 --- a/src/query/sql/src/executor/physical_plans/physical_row_fetch.rs +++ b/src/query/sql/src/executor/physical_plans/physical_row_fetch.rs @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::any::Any; + use databend_common_ast::ast::FormatTreeNode; use databend_common_catalog::plan::DataSourcePlan; use databend_common_catalog::plan::Projection; @@ -48,6 +50,9 @@ pub struct RowFetch { #[typetag::serde] impl IPhysicalPlan for RowFetch { + fn as_any(&self) -> &dyn Any { + self + } fn get_meta(&self) -> &PhysicalPlanMeta { &self.meta } diff --git a/src/query/sql/src/executor/physical_plans/physical_sort.rs b/src/query/sql/src/executor/physical_plans/physical_sort.rs index d0d191dd85415..aafe56a4907bc 100644 --- a/src/query/sql/src/executor/physical_plans/physical_sort.rs +++ b/src/query/sql/src/executor/physical_plans/physical_sort.rs @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::any::Any; + use databend_common_ast::ast::FormatTreeNode; use databend_common_catalog::plan::DataSourcePlan; use databend_common_exception::Result; @@ -59,6 +61,9 @@ pub struct Sort { #[typetag::serde] impl IPhysicalPlan for Sort { + fn as_any(&self) -> &dyn Any { + self + } fn get_meta(&self) -> &PhysicalPlanMeta { &self.meta } diff --git a/src/query/sql/src/executor/physical_plans/physical_table_scan.rs b/src/query/sql/src/executor/physical_plans/physical_table_scan.rs index b2b205619f4e2..6771a38cdfa6c 100644 --- a/src/query/sql/src/executor/physical_plans/physical_table_scan.rs +++ b/src/query/sql/src/executor/physical_plans/physical_table_scan.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::any::Any; use std::collections::btree_map::Entry; use std::collections::BTreeMap; use std::collections::HashMap; @@ -98,6 +99,9 @@ pub struct TableScan { #[typetag::serde] impl IPhysicalPlan for TableScan { + fn as_any(&self) -> &dyn Any { + self + } fn get_meta(&self) -> &PhysicalPlanMeta { &self.meta } diff --git a/src/query/sql/src/executor/physical_plans/physical_udf.rs b/src/query/sql/src/executor/physical_plans/physical_udf.rs index 135f1a3cfcb44..15ecbcf08607b 100644 --- a/src/query/sql/src/executor/physical_plans/physical_udf.rs +++ b/src/query/sql/src/executor/physical_plans/physical_udf.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::any::Any; use std::collections::BTreeMap; use databend_common_ast::ast::FormatTreeNode; @@ -53,6 +54,9 @@ pub struct Udf { #[typetag::serde] impl IPhysicalPlan for Udf { + fn as_any(&self) -> &dyn Any { + self + } fn get_meta(&self) -> &PhysicalPlanMeta { &self.meta } diff --git a/src/query/sql/src/executor/physical_plans/physical_union_all.rs b/src/query/sql/src/executor/physical_plans/physical_union_all.rs index a8eb52d30cb07..1108f28611135 100644 --- a/src/query/sql/src/executor/physical_plans/physical_union_all.rs +++ b/src/query/sql/src/executor/physical_plans/physical_union_all.rs @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::any::Any; + use databend_common_ast::ast::FormatTreeNode; use databend_common_exception::Result; use databend_common_expression::DataField; @@ -52,6 +54,9 @@ pub struct UnionAll { #[typetag::serde] impl IPhysicalPlan for UnionAll { + fn as_any(&self) -> &dyn Any { + self + } fn get_meta(&self) -> &PhysicalPlanMeta { &self.meta } diff --git a/src/query/sql/src/executor/physical_plans/physical_window.rs b/src/query/sql/src/executor/physical_plans/physical_window.rs index da5ee8bf5a51b..b716f51caf875 100644 --- a/src/query/sql/src/executor/physical_plans/physical_window.rs +++ b/src/query/sql/src/executor/physical_plans/physical_window.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::any::Any; use std::fmt::Display; use databend_common_ast::ast::FormatTreeNode; @@ -68,6 +69,9 @@ pub struct Window { #[typetag::serde] impl IPhysicalPlan for Window { + fn as_any(&self) -> &dyn Any { + self + } fn get_meta(&self) -> &PhysicalPlanMeta { &self.meta } diff --git a/src/query/sql/src/executor/physical_plans/physical_window_partition.rs b/src/query/sql/src/executor/physical_plans/physical_window_partition.rs index 0939f27ad1545..75ea095fab111 100644 --- a/src/query/sql/src/executor/physical_plans/physical_window_partition.rs +++ b/src/query/sql/src/executor/physical_plans/physical_window_partition.rs @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::any::Any; + use databend_common_ast::ast::FormatTreeNode; use databend_common_catalog::plan::DataSourcePlan; use databend_common_exception::Result; @@ -42,6 +44,9 @@ pub struct WindowPartition { #[typetag::serde] impl IPhysicalPlan for WindowPartition { + fn as_any(&self) -> &dyn Any { + self + } fn get_meta(&self) -> &PhysicalPlanMeta { &self.meta } From 8eb113919c30c07b5333b5272f2a70651caddf73 Mon Sep 17 00:00:00 2001 From: zhang2014 Date: Thu, 3 Jul 2025 16:36:57 +0800 Subject: [PATCH 8/8] refactor(query): use trait to refactor physical plan --- .../interpreter_copy_into_location.rs | 10 +- .../interpreter_copy_into_table.rs | 42 +-- .../src/interpreters/interpreter_explain.rs | 4 +- .../interpreters/interpreter_index_refresh.rs | 39 +-- .../src/interpreters/interpreter_insert.rs | 58 ++-- .../src/interpreters/interpreter_mutation.rs | 15 +- .../src/interpreters/interpreter_replace.rs | 95 +++--- .../src/interpreters/interpreter_select.rs | 18 +- .../interpreters/interpreter_table_analyze.rs | 94 ++---- .../interpreter_table_modify_column.rs | 29 +- .../interpreter_table_recluster.rs | 96 +++--- src/query/service/src/lib.rs | 1 + .../service/src/pipelines/pipeline_builder.rs | 8 +- .../transform_recursive_cte_source.rs | 3 +- .../src/schedulers/fragments/fragmenter.rs | 286 +++++++++--------- .../src/schedulers/fragments/plan_fragment.rs | 24 +- .../fragments/query_fragment_actions.rs | 6 +- .../query_fragment_actions_display.rs | 15 +- src/query/service/src/schedulers/scheduler.rs | 11 +- .../flight/v1/exchange/exchange_manager.rs | 4 +- .../flight/v1/packets/packet_fragment.rs | 2 +- .../it/pipelines/builders/runtime_filter.rs | 4 +- src/query/sql/src/executor/mod.rs | 1 + src/query/sql/src/executor/physical_plan.rs | 48 +-- .../sql/src/executor/physical_plan_visitor.rs | 58 +--- .../physical_plans/physical_table_scan.rs | 2 +- 26 files changed, 418 insertions(+), 555 deletions(-) diff --git a/src/query/service/src/interpreters/interpreter_copy_into_location.rs b/src/query/service/src/interpreters/interpreter_copy_into_location.rs index 5026095e51a08..23877fb5fc6e4 100644 --- a/src/query/service/src/interpreters/interpreter_copy_into_location.rs +++ b/src/query/service/src/interpreters/interpreter_copy_into_location.rs @@ -20,7 +20,7 @@ use databend_common_expression::infer_table_schema; use databend_common_meta_app::schema::UpdateStreamMetaReq; use databend_common_pipeline_core::ExecutionInfo; use databend_common_sql::executor::physical_plans::CopyIntoLocation; -use databend_common_sql::executor::PhysicalPlan; +use databend_common_sql::executor::{IPhysicalPlan, PhysicalPlan, PhysicalPlanMeta}; use databend_storages_common_stage::CopyIntoLocationInfo; use log::debug; use log::info; @@ -89,14 +89,14 @@ impl CopyIntoLocationInterpreter { let query_result_schema = query_interpreter.get_result_schema(); let table_schema = infer_table_schema(&query_result_schema)?; - let mut physical_plan = PhysicalPlan::CopyIntoLocation(Box::new(CopyIntoLocation { - plan_id: 0, - input: Box::new(query_physical_plan), + let mut physical_plan: Box = Box::new(CopyIntoLocation { + input: query_physical_plan, project_columns: query_interpreter.get_result_columns(), input_data_schema: query_result_schema, input_table_schema: table_schema, info: info.clone(), - })); + meta: PhysicalPlanMeta::new("CopyIntoLocation"), + }); let mut next_plan_id = 0; physical_plan.adjust_plan_id(&mut next_plan_id); diff --git a/src/query/service/src/interpreters/interpreter_copy_into_table.rs b/src/query/service/src/interpreters/interpreter_copy_into_table.rs index 4c5694151dee6..ad52b4e1e431c 100644 --- a/src/query/service/src/interpreters/interpreter_copy_into_table.rs +++ b/src/query/service/src/interpreters/interpreter_copy_into_table.rs @@ -33,7 +33,7 @@ use databend_common_sql::executor::physical_plans::FragmentKind; use databend_common_sql::executor::physical_plans::MutationKind; use databend_common_sql::executor::physical_plans::TableScan; use databend_common_sql::executor::table_read_plan::ToReadDataSourcePlan; -use databend_common_sql::executor::PhysicalPlan; +use databend_common_sql::executor::{IPhysicalPlan, PhysicalPlan, PhysicalPlanMeta}; use databend_common_storage::StageFileInfo; use databend_common_storages_fuse::FuseTable; use databend_common_storages_stage::StageTable; @@ -100,7 +100,7 @@ impl CopyIntoTableInterpreter { &self, table_info: TableInfo, plan: &CopyIntoTablePlan, - ) -> Result<(PhysicalPlan, Vec)> { + ) -> Result<(Box, Vec)> { let to_table = self .ctx .get_table( @@ -125,7 +125,7 @@ impl CopyIntoTableInterpreter { let (query_interpreter, update_stream_meta) = self.build_query(&query).await?; update_stream_meta_reqs = update_stream_meta; - let query_physical_plan = Box::new(query_interpreter.build_physical_plan().await?); + let query_physical_plan = query_interpreter.build_physical_plan().await?; let result_columns = query_interpreter.get_result_columns(); ( @@ -145,21 +145,20 @@ impl CopyIntoTableInterpreter { } ( - CopyIntoTableSource::Stage(Box::new(PhysicalPlan::TableScan(TableScan { - plan_id: 0, + CopyIntoTableSource::Stage(Box::new(TableScan { scan_id: 0, name_mapping, stat_info: None, table_index: None, internal_column: None, source: Box::new(data_source_plan), - }))), + meta: PhysicalPlanMeta::new("TableScan"), + })), None, ) }; - let mut root = PhysicalPlan::CopyIntoTable(Box::new(CopyIntoTable { - plan_id: 0, + let mut root: Box = Box::new(CopyIntoTable { required_values_schema: plan.required_values_schema.clone(), values_consts: plan.values_consts.clone(), required_source_schema: plan.required_source_schema.clone(), @@ -171,16 +170,17 @@ impl CopyIntoTableInterpreter { source, is_transform: plan.is_transform, table_meta_timestamps, - })); + meta: PhysicalPlanMeta::new("CopyIntoTable"), + }); if plan.enable_distributed { - root = PhysicalPlan::Exchange(Exchange { - plan_id: 0, - input: Box::new(root), + root = Box::new(Exchange { + input: root, kind: FragmentKind::Merge, keys: Vec::new(), allow_adjust_parallelism: true, ignore_exchange: false, + meta: PhysicalPlanMeta::new("Exchange"), }); } @@ -318,14 +318,14 @@ impl CopyIntoTableInterpreter { if self.plan.stage_table_info.copy_into_table_options.purge && !self - .plan - .stage_table_info - .duplicated_files_detected - .is_empty() + .plan + .stage_table_info + .duplicated_files_detected + .is_empty() && self - .ctx - .get_settings() - .get_enable_purge_duplicated_files_in_copy()? + .ctx + .get_settings() + .get_enable_purge_duplicated_files_in_copy()? { info!( "purge_duplicated_files_in_copy enabled, number of duplicated files: {}", @@ -337,7 +337,7 @@ impl CopyIntoTableInterpreter { self.plan.stage_table_info.duplicated_files_detected.clone(), self.plan.stage_table_info.stage_info.clone(), ) - .await?; + .await?; } Ok(PipelineBuildResult::create()) } @@ -406,7 +406,7 @@ impl Interpreter for CopyIntoTableInterpreter { unsafe { self.ctx.get_settings().get_deduplicate_label()? }, self.plan.path_prefix.clone(), ) - .await?; + .await?; } // Execute hook. diff --git a/src/query/service/src/interpreters/interpreter_explain.rs b/src/query/service/src/interpreters/interpreter_explain.rs index 6a15a1046bc57..7cb468961e73d 100644 --- a/src/query/service/src/interpreters/interpreter_explain.rs +++ b/src/query/service/src/interpreters/interpreter_explain.rs @@ -34,7 +34,6 @@ use databend_common_pipeline_core::processors::PlanProfile; use databend_common_pipeline_core::ExecutionInfo; use databend_common_pipeline_core::Pipeline; use databend_common_sql::binder::ExplainConfig; -use databend_common_sql::executor::format_partial_tree; use databend_common_sql::executor::IPhysicalPlan; use databend_common_sql::executor::MutationBuildInfo; use databend_common_sql::plans::Mutation; @@ -69,6 +68,7 @@ use crate::sql::executor::PhysicalPlan; use crate::sql::executor::PhysicalPlanBuilder; use crate::sql::optimizer::ir::SExpr; use crate::sql::plans::Plan; +use databend_common_sql::executor::PhysicalPlanDynExt; pub struct ExplainInterpreter { ctx: Arc, @@ -565,7 +565,7 @@ impl ExplainInterpreter { fn inject_pruned_partitions_stats( &self, - plan: &mut PhysicalPlan, + plan: &mut Box, metadata: &MetadataRef, ) -> Result<()> { let mut sources = vec![]; diff --git a/src/query/service/src/interpreters/interpreter_index_refresh.rs b/src/query/service/src/interpreters/interpreter_index_refresh.rs index 7b687d38a385e..a5bdb339a853e 100644 --- a/src/query/service/src/interpreters/interpreter_index_refresh.rs +++ b/src/query/service/src/interpreters/interpreter_index_refresh.rs @@ -35,7 +35,7 @@ use databend_common_pipeline_transforms::processors::TransformPipelineHelper; use databend_common_sql::evaluator::BlockOperator; use databend_common_sql::evaluator::CompoundBlockOperator; use databend_common_sql::executor::physical_plans::TableScan; -use databend_common_sql::executor::DeriveHandle; +use databend_common_sql::executor::{DeriveHandle, PhysicalPlanDynExt}; use databend_common_sql::executor::IPhysicalPlan; use databend_common_sql::executor::PhysicalPlan; use databend_common_sql::executor::PhysicalPlanBuilder; @@ -55,6 +55,7 @@ use crate::interpreters::Interpreter; use crate::pipelines::PipelineBuildResult; use crate::schedulers::build_query_pipeline_without_render_result_set; use crate::sessions::QueryContext; +use crate::test_kits::query_count; pub struct RefreshIndexInterpreter { ctx: Arc, @@ -121,32 +122,20 @@ impl RefreshIndexInterpreter { #[async_backtrace::framed] async fn get_read_source( &self, - query_plan: &PhysicalPlan, + query_plan: &Box, fuse_table: Arc, segments: Option>, ) -> Result> { - let mut source = vec![]; + let mut sources = vec![]; + query_plan.get_all_data_source(&mut sources); - let mut collect_read_source = |plan: &PhysicalPlan| { - if let PhysicalPlan::TableScan(scan) = plan { - source.push(*scan.source.clone()) - } - }; - - PhysicalPlan::traverse( - query_plan, - &mut |_| true, - &mut collect_read_source, - &mut |_| {}, - ); - - if source.len() != 1 { + if sources.len() != 1 { Err(ErrorCode::Internal( "Invalid source with multiple table scan when do refresh aggregating index" .to_string(), )) } else { - let mut source = source.remove(0); + let (_, mut source) = sources.remove(0); let partitions = match segments { Some(segment_locs) if !segment_locs.is_empty() => { let segment_locations = create_segment_location_vector(segment_locs, None); @@ -188,7 +177,7 @@ impl RefreshIndexInterpreter { }; if !source.parts.is_empty() { - Ok(Some(source)) + Ok(Some(Box::into_inner(source))) } else { Ok(None) } @@ -279,10 +268,8 @@ impl Interpreter for RefreshIndexInterpreter { let new_index_meta = self.update_index_meta(&new_read_source)?; - let mut replace_read_source = ReadSourceReplacer { - source: new_read_source, - }; - query_plan = replace_read_source.replace(&query_plan)?; + let mut handle = ReadSourceDeriveHandle::new(new_read_source); + query_plan = query_plan.derive_with(&mut handle); let mut build_res = build_query_pipeline_without_render_result_set(&self.ctx, &query_plan).await?; @@ -382,6 +369,12 @@ struct ReadSourceDeriveHandle { source: DataSourcePlan, } +impl ReadSourceDeriveHandle { + pub fn new(source:DataSourcePlan) -> Box { + Box::new(ReadSourceDeriveHandle { source }) + } +} + impl DeriveHandle for ReadSourceDeriveHandle { fn derive( &mut self, diff --git a/src/query/service/src/interpreters/interpreter_insert.rs b/src/query/service/src/interpreters/interpreter_insert.rs index e45615a8ae5f3..27bff610409e9 100644 --- a/src/query/service/src/interpreters/interpreter_insert.rs +++ b/src/query/service/src/interpreters/interpreter_insert.rs @@ -25,9 +25,9 @@ use databend_common_expression::FromData; use databend_common_expression::SendableDataBlockStream; use databend_common_pipeline_sources::AsyncSourcer; use databend_common_pipeline_transforms::TransformPipelineHelper; -use databend_common_sql::executor::physical_plans::DistributedInsertSelect; +use databend_common_sql::executor::physical_plans::{DistributedInsertSelect, Exchange}; use databend_common_sql::executor::physical_plans::MutationKind; -use databend_common_sql::executor::PhysicalPlan; +use databend_common_sql::executor::{IPhysicalPlan, PhysicalPlan, PhysicalPlanDynExt, PhysicalPlanMeta}; use databend_common_sql::executor::PhysicalPlanBuilder; use databend_common_sql::plans::Insert; use databend_common_sql::plans::InsertInputSource; @@ -174,55 +174,49 @@ impl Interpreter for InsertInterpreter { _ => unreachable!(), }; - let explain_plan = select_plan - .format(metadata.clone(), Default::default())? - .format_pretty()?; + let explain_plan = { + let metadata = metadata.read(); + select_plan.format(&metadata, Default::default())?.format_pretty()? + }; + info!("Insert select plan: \n{}", explain_plan); let update_stream_meta = dml_build_update_stream_req(self.ctx.clone()).await?; // here we remove the last exchange merge plan to trigger distribute insert - let insert_select_plan = match (select_plan, table.support_distributed_insert()) { - (PhysicalPlan::Exchange(ref mut exchange), true) => { + let mut insert_select_plan = { + if table.support_distributed_insert() && let Some(exchange) = select_plan.downcast_ref::() { // insert can be dispatched to different nodes if table support_distributed_insert let input = exchange.input.clone(); - - exchange.input = Box::new(PhysicalPlan::DistributedInsertSelect(Box::new( - DistributedInsertSelect { - // TODO(leiysky): we reuse the id of exchange here, - // which is not correct. We should generate a new id for insert. - plan_id: exchange.plan_id, - input, - table_info: table1.get_table_info().clone(), - select_schema: plan.schema(), - select_column_bindings, - insert_schema: self.plan.dest_schema(), - cast_needed: self.check_schema_cast(plan)?, - table_meta_timestamps, - }, - ))); - PhysicalPlan::Exchange(exchange.clone()) - } - (other_plan, _) => { + exchange.derive(vec![Box::new(DistributedInsertSelect { + input, + table_info: table1.get_table_info().clone(), + select_schema: plan.schema(), + select_column_bindings, + insert_schema: self.plan.dest_schema(), + cast_needed: self.check_schema_cast(plan)?, + table_meta_timestamps, + meta: PhysicalPlanMeta::new("DistributedInsertSelect"), + })]) + } else { // insert should wait until all nodes finished - PhysicalPlan::DistributedInsertSelect(Box::new(DistributedInsertSelect { + Box::new(DistributedInsertSelect { // TODO: we reuse the id of other plan here, // which is not correct. We should generate a new id for insert. - plan_id: other_plan.get_id(), - input: Box::new(other_plan), + input: select_plan, table_info: table1.get_table_info().clone(), select_schema: plan.schema(), select_column_bindings, insert_schema: self.plan.dest_schema(), cast_needed: self.check_schema_cast(plan)?, table_meta_timestamps, - })) + meta: PhysicalPlanMeta::new("DistributedInsertSelect"), + }) } }; - let mut build_res = - build_query_pipeline_without_render_result_set(&self.ctx, &insert_select_plan) - .await?; + insert_select_plan.adjust_plan_id(&mut 0); + let mut build_res = build_query_pipeline_without_render_result_set(&self.ctx, &insert_select_plan).await?; table.commit_insertion( self.ctx.clone(), diff --git a/src/query/service/src/interpreters/interpreter_mutation.rs b/src/query/service/src/interpreters/interpreter_mutation.rs index 023929151ef1f..d6f0aeedc5dd3 100644 --- a/src/query/service/src/interpreters/interpreter_mutation.rs +++ b/src/query/service/src/interpreters/interpreter_mutation.rs @@ -32,7 +32,7 @@ use databend_common_sql::binder::MutationStrategy; use databend_common_sql::binder::MutationType; use databend_common_sql::executor::physical_plans::create_push_down_filters; use databend_common_sql::executor::physical_plans::MutationKind; -use databend_common_sql::executor::IPhysicalPlan; +use databend_common_sql::executor::{IPhysicalPlan, PhysicalPlanDynExt}; use databend_common_sql::executor::MutationBuildInfo; use databend_common_sql::executor::PhysicalPlan; use databend_common_sql::executor::PhysicalPlanBuilder; @@ -99,9 +99,10 @@ impl Interpreter for MutationInterpreter { // Build physical plan. let physical_plan = self.build_physical_plan(&mutation, false).await?; - let query_plan = physical_plan - .format(self.metadata.clone(), Default::default())? - .format_pretty()?; + let query_plan = { + let metadata = self.metadata.read(); + physical_plan.format(&metadata, Default::default())?.format_pretty()? + }; info!("Query physical plan: \n{}", query_plan); @@ -231,7 +232,7 @@ pub async fn build_mutation_info( table_snapshot.clone(), dry_run, ) - .await?; + .await?; let table_meta_timestamps = ctx.get_table_meta_timestamps(table.as_ref(), table_snapshot.clone())?; @@ -283,8 +284,8 @@ async fn mutation_source_partitions( let cluster = ctx.get_cluster(); let is_lazy = fuse_table.is_column_oriented() || (!dry_run - && !cluster.is_empty() - && table_snapshot.segments.len() >= cluster.nodes.len()); + && !cluster.is_empty() + && table_snapshot.segments.len() >= cluster.nodes.len()); (is_lazy, true) } else { (false, false) diff --git a/src/query/service/src/interpreters/interpreter_replace.rs b/src/query/service/src/interpreters/interpreter_replace.rs index efbde608318cd..c1cb946e29673 100644 --- a/src/query/service/src/interpreters/interpreter_replace.rs +++ b/src/query/service/src/interpreters/interpreter_replace.rs @@ -1,5 +1,3 @@ -// Copyright 2021 Datafuse Labs -// // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -25,7 +23,7 @@ use databend_common_functions::BUILTIN_FUNCTIONS; use databend_common_meta_app::principal::StageInfo; use databend_common_meta_app::schema::TableInfo; use databend_common_meta_app::schema::UpdateStreamMetaReq; -use databend_common_sql::executor::cast_expr_to_non_null_boolean; +use databend_common_sql::executor::{cast_expr_to_non_null_boolean, IPhysicalPlan, PhysicalPlanDynExt, PhysicalPlanMeta}; use databend_common_sql::executor::physical_plans::CommitSink; use databend_common_sql::executor::physical_plans::CommitType; use databend_common_sql::executor::physical_plans::Exchange; @@ -133,7 +131,7 @@ impl ReplaceInterpreter { async fn build_physical_plan( &self, ) -> Result<( - Box, + Box, Option<(Vec, StageInfo, CopyIntoTableOptions)>, )> { let plan = &self.plan; @@ -264,34 +262,29 @@ impl ReplaceInterpreter { let mut is_exchange = false; let is_stage_source = matches!(self.plan.source, InsertInputSource::Stage(_)); - if let PhysicalPlan::Exchange(Exchange { - input, - kind: FragmentKind::Merge, - .. - }) = root.as_ref() - { + if let Some(exchange) = root.downcast_ref::() && exchange.kind == FragmentKind::Merge { is_exchange = true; - root = input.clone(); + root = exchange.input.clone(); } if is_distributed { - root = Box::new(PhysicalPlan::Exchange(Exchange { - plan_id: 0, + root = Box::new(Exchange { input: root, kind: FragmentKind::Expansive, keys: vec![], allow_adjust_parallelism: true, ignore_exchange: false, - })); + meta: PhysicalPlanMeta::new("Exchange"), + }); } else if is_exchange && !is_stage_source { - root = Box::new(PhysicalPlan::Exchange(Exchange { - plan_id: 0, + root = Box::new(Exchange { input: root, kind: FragmentKind::Merge, keys: vec![], allow_adjust_parallelism: true, ignore_exchange: false, - })); + meta: PhysicalPlanMeta::new("Exchange"), + }); } let max_num_pruning_columns = self @@ -313,23 +306,21 @@ impl ReplaceInterpreter { vec![] }; - root = Box::new(PhysicalPlan::ReplaceDeduplicate(Box::new( - ReplaceDeduplicate { - input: root, - on_conflicts: on_conflicts.clone(), - bloom_filter_column_indexes: bloom_filter_column_indexes.clone(), - table_is_empty, - table_info: table_info.clone(), - select_ctx, - target_schema: plan.schema.clone(), - table_level_range_index, - need_insert: true, - delete_when, - plan_id: u32::MAX, - }, - ))); + root = Box::new(ReplaceDeduplicate { + input: root, + on_conflicts: on_conflicts.clone(), + bloom_filter_column_indexes: bloom_filter_column_indexes.clone(), + table_is_empty, + table_info: table_info.clone(), + select_ctx, + target_schema: plan.schema.clone(), + table_level_range_index, + need_insert: true, + delete_when, + meta: PhysicalPlanMeta::new("ReplaceDeduplicate"), + }); - root = Box::new(PhysicalPlan::ReplaceInto(Box::new(ReplaceInto { + root = Box::new(ReplaceInto { input: root, block_thresholds: fuse_table.get_block_thresholds(), table_info: table_info.clone(), @@ -343,22 +334,22 @@ impl ReplaceInterpreter { .collect(), block_slots: None, need_insert: true, - plan_id: u32::MAX, table_meta_timestamps, - }))); + meta: PhysicalPlanMeta::new("ReplaceInto"), + }); if is_distributed { - root = Box::new(PhysicalPlan::Exchange(Exchange { - plan_id: 0, + root = Box::new(Exchange { input: root, kind: FragmentKind::Merge, keys: vec![], allow_adjust_parallelism: true, ignore_exchange: false, - })); + meta: PhysicalPlanMeta::new("Exchange"), + }); } - root = Box::new(PhysicalPlan::CommitSink(Box::new(CommitSink { + root = Box::new(CommitSink { input: root, snapshot: base_snapshot, table_info: table_info.clone(), @@ -368,10 +359,11 @@ impl ReplaceInterpreter { }, update_stream_meta: update_stream_meta.clone(), deduplicated_label: unsafe { self.ctx.get_settings().get_deduplicate_label()? }, - plan_id: u32::MAX, table_meta_timestamps, recluster_info: None, - }))); + meta: PhysicalPlanMeta::new("CommitSink"), + }); + root.adjust_plan_id(&mut 0); Ok((root, purge_info)) } @@ -424,7 +416,7 @@ impl ReplaceInterpreter { copy_plan.stage_table_info.copy_into_table_options.clone(), )); Ok(ReplaceSourceCtx { - root: Box::new(physical_plan), + root: physical_plan, select_ctx: None, update_stream_meta: vec![], bind_context: None, @@ -442,14 +434,12 @@ impl ReplaceInterpreter { &self, schema: DataSchemaRef, source: &InsertValue, - ) -> Result> { - Ok(Box::new(PhysicalPlan::ReplaceAsyncSourcer( - ReplaceAsyncSourcer { - schema, - plan_id: u32::MAX, - source: source.clone(), - }, - ))) + ) -> Result> { + Ok(Box::new(ReplaceAsyncSourcer { + schema, + source: source.clone(), + meta: PhysicalPlanMeta::new("ReplaceAsyncSourcer"), + })) } #[async_backtrace::framed] @@ -482,8 +472,7 @@ impl ReplaceInterpreter { let physical_plan = select_interpreter .build_physical_plan() - .await - .map(Box::new)?; + .await?; let select_ctx = ReplaceSelectCtx { select_column_bindings: bind_context.columns.clone(), select_schema: query_plan.schema(), @@ -498,7 +487,7 @@ impl ReplaceInterpreter { } struct ReplaceSourceCtx { - root: Box, + root: Box, select_ctx: Option, update_stream_meta: Vec, bind_context: Option, diff --git a/src/query/service/src/interpreters/interpreter_select.rs b/src/query/service/src/interpreters/interpreter_select.rs index 73e43fc90f51f..5f434eff91720 100644 --- a/src/query/service/src/interpreters/interpreter_select.rs +++ b/src/query/service/src/interpreters/interpreter_select.rs @@ -32,8 +32,8 @@ use databend_common_pipeline_core::Pipe; use databend_common_pipeline_core::PipeItem; use databend_common_pipeline_core::Pipeline; use databend_common_pipeline_transforms::processors::TransformDummy; -use databend_common_sql::executor::physical_plans::FragmentKind; -use databend_common_sql::executor::IPhysicalPlan; +use databend_common_sql::executor::physical_plans::{Exchange, FragmentKind}; +use databend_common_sql::executor::{IPhysicalPlan, PhysicalPlanDynExt}; use databend_common_sql::executor::PhysicalPlan; use databend_common_sql::parse_result_scan_args; use databend_common_sql::ColumnBinding; @@ -123,9 +123,9 @@ impl SelectInterpreter { #[async_backtrace::framed] pub async fn build_pipeline( &self, - mut physical_plan: PhysicalPlan, + mut physical_plan: Box, ) -> Result { - if let PhysicalPlan::Exchange(exchange) = &mut physical_plan { + if let Some(exchange) = physical_plan.downcast_mut_ref::() { if exchange.kind == FragmentKind::Merge && self.ignore_result { exchange.ignore_exchange = self.ignore_result; } @@ -137,7 +137,7 @@ impl SelectInterpreter { &physical_plan, self.ignore_result, ) - .await?; + .await?; // consume stream let update_stream_metas = query_build_update_stream_req(&self.ctx).await?; @@ -286,9 +286,11 @@ impl Interpreter for SelectInterpreter { // 0. Need to build physical plan first to get the partitions. let physical_plan = self.build_physical_plan().await?; - let query_plan = physical_plan - .format(self.metadata.clone(), Default::default())? - .format_pretty()?; + + let query_plan = { + let metadata = self.metadata.read(); + physical_plan.format(&metadata, Default::default())?.format_pretty()? + }; info!("[SELECT-INTERP] Query physical plan:\n{}", query_plan); diff --git a/src/query/service/src/interpreters/interpreter_table_analyze.rs b/src/query/service/src/interpreters/interpreter_table_analyze.rs index d9dc116c15ea5..875b63c72d1a8 100644 --- a/src/query/service/src/interpreters/interpreter_table_analyze.rs +++ b/src/query/service/src/interpreters/interpreter_table_analyze.rs @@ -19,14 +19,14 @@ use chrono::Utc; use databend_common_catalog::table::TableExt; use databend_common_exception::Result; use databend_common_pipeline_core::processors::ProcessorPtr; -use databend_common_sql::executor::physical_plans::AggregateExpand; +use databend_common_sql::executor::physical_plans::{AggregateExpand, Exchange}; use databend_common_sql::executor::physical_plans::AggregateFinal; use databend_common_sql::executor::physical_plans::AggregatePartial; use databend_common_sql::executor::physical_plans::EvalScalar; use databend_common_sql::executor::physical_plans::Filter; use databend_common_sql::executor::physical_plans::Sort; use databend_common_sql::executor::physical_plans::Window; -use databend_common_sql::executor::PhysicalPlan; +use databend_common_sql::executor::{DeriveHandle, IPhysicalPlan, PhysicalPlan, PhysicalPlanDynExt}; use databend_common_sql::executor::PhysicalPlanBuilder; use databend_common_sql::plans::AnalyzeTablePlan; use databend_common_sql::plans::Plan; @@ -59,7 +59,7 @@ impl AnalyzeTableInterpreter { Ok(AnalyzeTableInterpreter { ctx, plan }) } - async fn plan_sql(&self, sql: String) -> Result<(PhysicalPlan, BindContext)> { + async fn plan_sql(&self, sql: String) -> Result<(Box, BindContext)> { let mut planner = Planner::new(self.ctx.clone()); let (plan, _) = planner.plan_sql(&sql).await?; let (select_plan, bind_context) = match &plan { @@ -234,7 +234,7 @@ impl Interpreter for AnalyzeTableInterpreter { &histogram_plan, false, ) - .await?; + .await?; let (tx, rx) = async_channel::unbounded(); histogram_build_res.main_pipeline.add_sink(|input_port| { Ok(ProcessorPtr::create(HistogramInfoSink::create( @@ -269,76 +269,24 @@ impl Interpreter for AnalyzeTableInterpreter { } } -fn remove_exchange(plan: PhysicalPlan) -> PhysicalPlan { - #[recursive::recursive] - fn traverse(plan: PhysicalPlan) -> PhysicalPlan { - match plan { - PhysicalPlan::Filter(plan) => PhysicalPlan::Filter(Filter { - plan_id: plan.plan_id, - projections: plan.projections, - input: Box::new(traverse(*plan.input)), - predicates: plan.predicates, - stat_info: plan.stat_info, - }), - PhysicalPlan::EvalScalar(plan) => PhysicalPlan::EvalScalar(EvalScalar { - plan_id: plan.plan_id, - projections: plan.projections, - input: Box::new(traverse(*plan.input)), - exprs: plan.exprs, - stat_info: plan.stat_info, - }), - PhysicalPlan::AggregateExpand(plan) => PhysicalPlan::AggregateExpand(AggregateExpand { - plan_id: plan.plan_id, - input: Box::new(traverse(*plan.input)), - group_bys: plan.group_bys, - grouping_sets: plan.grouping_sets, - stat_info: plan.stat_info, - }), - PhysicalPlan::AggregatePartial(plan) => { - PhysicalPlan::AggregatePartial(AggregatePartial { - plan_id: plan.plan_id, - input: Box::new(traverse(*plan.input)), - group_by: plan.group_by, - agg_funcs: plan.agg_funcs, - rank_limit: plan.rank_limit, - enable_experimental_aggregate_hashtable: plan - .enable_experimental_aggregate_hashtable, - group_by_display: plan.group_by_display, - stat_info: plan.stat_info, - }) - } - PhysicalPlan::AggregateFinal(plan) => PhysicalPlan::AggregateFinal(AggregateFinal { - plan_id: plan.plan_id, - input: Box::new(traverse(*plan.input)), - group_by: plan.group_by, - agg_funcs: plan.agg_funcs, - before_group_by_schema: plan.before_group_by_schema, - group_by_display: plan.group_by_display, - stat_info: plan.stat_info, - }), - PhysicalPlan::Window(plan) => PhysicalPlan::Window(Window { - plan_id: plan.plan_id, - index: plan.index, - input: Box::new(traverse(*plan.input)), - func: plan.func, - partition_by: plan.partition_by, - order_by: plan.order_by, - window_frame: plan.window_frame, - limit: plan.limit, - }), - PhysicalPlan::Sort(plan) => PhysicalPlan::Sort(Sort { - plan_id: plan.plan_id, - input: Box::new(traverse(*plan.input)), - order_by: plan.order_by, - limit: plan.limit, - after_exchange: plan.after_exchange, - pre_projection: plan.pre_projection, - stat_info: plan.stat_info, - }), - PhysicalPlan::Exchange(plan) => traverse(*plan.input), - _ => plan, +fn remove_exchange(plan: Box) -> Box { + struct RemoveExchangeHandle; + + impl DeriveHandle for RemoveExchangeHandle { + fn derive( + &mut self, + v: &Box, + mut children: Vec>, + ) -> std::result::Result, Vec>> { + let Some(_) = v.downcast_ref::() else { + return Err(children); + }; + + assert_eq!(children.len(), 1); + Ok(children.remove(0)) } } - traverse(plan) + let mut handle = Box::new(RemoveExchangeHandle); + plan.derive_with(&mut handle) } diff --git a/src/query/service/src/interpreters/interpreter_table_modify_column.rs b/src/query/service/src/interpreters/interpreter_table_modify_column.rs index 6e4af8c2e69a9..9d51d7433ebb9 100644 --- a/src/query/service/src/interpreters/interpreter_table_modify_column.rs +++ b/src/query/service/src/interpreters/interpreter_table_modify_column.rs @@ -39,7 +39,7 @@ use databend_common_meta_app::schema::TableMeta; use databend_common_meta_app::schema::UpdateTableMetaReq; use databend_common_meta_types::MatchSeq; use databend_common_sql::executor::physical_plans::DistributedInsertSelect; -use databend_common_sql::executor::PhysicalPlan; +use databend_common_sql::executor::{IPhysicalPlan, PhysicalPlan, PhysicalPlanMeta}; use databend_common_sql::executor::PhysicalPlanBuilder; use databend_common_sql::plans::ModifyColumnAction; use databend_common_sql::plans::ModifyTableColumnPlan; @@ -227,7 +227,7 @@ impl ModifyTableColumnInterpreter { for (index_name, index) in &table_info.meta.indexes { if index.column_ids.contains(&old_field.column_id) && old_field.data_type.remove_nullable() - != field.data_type.remove_nullable() + != field.data_type.remove_nullable() { return Err(ErrorCode::ColumnReferencedByInvertedIndex(format!( "column `{}` is referenced by inverted index, drop inverted index `{}` first", @@ -289,7 +289,7 @@ impl ModifyTableColumnInterpreter { table_info.meta.clone(), catalog, ) - .await?; + .await?; return Ok(PipelineBuildResult::create()); } @@ -452,7 +452,7 @@ impl ModifyTableColumnInterpreter { prev_snapshot_id, table_meta_timestamps, ) - .await + .await } // Set column comment. @@ -492,7 +492,7 @@ impl ModifyTableColumnInterpreter { table_info.meta.clone(), catalog, ) - .await?; + .await?; } Ok(PipelineBuildResult::create()) @@ -650,7 +650,7 @@ impl Interpreter for ModifyTableColumnInterpreter { table_meta, column.to_string(), ) - .await? + .await? } }; @@ -685,9 +685,9 @@ fn is_string_to_binary(old_ty: &TableDataType, new_ty: &TableDataType) -> bool { ) => { old_tys.len() == new_tys.len() && old_tys - .iter() - .zip(new_tys) - .all(|(old_ty, new_ty)| is_string_to_binary(old_ty, new_ty)) + .iter() + .zip(new_tys) + .all(|(old_ty, new_ty)| is_string_to_binary(old_ty, new_ty)) } _ => false, } @@ -727,16 +727,19 @@ pub(crate) async fn build_select_insert_plan( let new_table = FuseTable::try_create(table_info)?; // 4. build DistributedInsertSelect plan - let insert_plan = PhysicalPlan::DistributedInsertSelect(Box::new(DistributedInsertSelect { - plan_id: select_plan.get_id(), - input: Box::new(select_plan), + let mut insert_plan: Box = Box::new(DistributedInsertSelect { + input: select_plan, table_info: new_table.get_table_info().clone(), select_schema, select_column_bindings, insert_schema: Arc::new(new_schema.into()), cast_needed: true, table_meta_timestamps, - })); + meta: PhysicalPlanMeta::new("DistributedInsertSelect"), + }); + + let mut index = 0; + insert_plan.adjust_plan_id(&mut index); let mut build_res = build_query_pipeline_without_render_result_set(&ctx, &insert_plan).await?; // 5. commit new meta schema and snapshots diff --git a/src/query/service/src/interpreters/interpreter_table_recluster.rs b/src/query/service/src/interpreters/interpreter_table_recluster.rs index f3c53597b06d7..5f02d1ab21d64 100644 --- a/src/query/service/src/interpreters/interpreter_table_recluster.rs +++ b/src/query/service/src/interpreters/interpreter_table_recluster.rs @@ -37,7 +37,7 @@ use databend_common_meta_app::schema::TableInfo; use databend_common_pipeline_core::always_callback; use databend_common_pipeline_core::ExecutionInfo; use databend_common_sql::bind_table; -use databend_common_sql::executor::cast_expr_to_non_null_boolean; +use databend_common_sql::executor::{cast_expr_to_non_null_boolean, IPhysicalPlan, PhysicalPlanDynExt, PhysicalPlanMeta}; use databend_common_sql::executor::physical_plans::CommitSink; use databend_common_sql::executor::physical_plans::CommitType; use databend_common_sql::executor::physical_plans::CompactSource; @@ -299,7 +299,7 @@ impl ReclusterTableInterpreter { tbl: &Arc, push_downs: &mut Option, hilbert_info: &mut Option, - ) -> Result> { + ) -> Result>> { LicenseManagerSwitch::instance() .check_enterprise_enabled(self.ctx.get_license_key(), Feature::HilbertClustering)?; let handler = get_hilbert_clustering_handler(); @@ -330,11 +330,11 @@ impl ReclusterTableInterpreter { // Adjust number of partitions according to the block size thresholds if total_partitions < block_thresholds.block_per_segment && block_thresholds.check_perfect_segment( - block_thresholds.block_per_segment, // this effectively by-pass the total_blocks criteria - total_rows, - total_bytes, - total_compressed, - ) + block_thresholds.block_per_segment, // this effectively by-pass the total_blocks criteria + total_rows, + total_bytes, + total_compressed, + ) { total_partitions = block_thresholds.block_per_segment; } @@ -419,18 +419,13 @@ impl ReclusterTableInterpreter { metadata.write().replace_all_tables(tbl.clone()); let mut builder = PhysicalPlanBuilder::new(metadata.clone(), self.ctx.clone(), false); - let mut plan = Box::new(builder.build(&s_expr, bind_context.column_set()).await?); + let mut plan = builder.build(&s_expr, bind_context.column_set()).await?; // Check if the plan already has an exchange operator let mut is_exchange = false; - if let PhysicalPlan::Exchange(Exchange { - input, - kind: FragmentKind::Merge, - .. - }) = plan.as_ref() - { + if let Some(exchange) = plan.downcast_ref::() && exchange.kind == FragmentKind::Merge { is_exchange = true; - plan = input.clone(); + plan = exchange.input.clone(); } // Determine if we need distributed execution @@ -451,14 +446,14 @@ impl ReclusterTableInterpreter { // Add exchange operator for data distribution, // shuffling data based on the hash of range partition IDs derived from the Hilbert index. - plan = Box::new(PhysicalPlan::Exchange(Exchange { - plan_id: 0, + plan = Box::new(Exchange { input: plan, kind: FragmentKind::Normal, keys: vec![expr], allow_adjust_parallelism: true, ignore_exchange: false, - })); + meta: PhysicalPlanMeta::new("Exchange"), + }); } let table_meta_timestamps = self @@ -467,14 +462,15 @@ impl ReclusterTableInterpreter { // Create the Hilbert partition physical plan, // collecting data into partitions and persist them - let plan = PhysicalPlan::HilbertPartition(Box::new(HilbertPartition { - plan_id: 0, + plan = Box::new(HilbertPartition { + rows_per_block, + table_meta_timestamps, + input: plan, table_info: table_info.clone(), num_partitions: total_partitions, - table_meta_timestamps, - rows_per_block, - })); + meta: PhysicalPlanMeta::new("HilbertPartition"), + }); // Finally, commit the newly clustered table Ok(Some(Self::add_commit_sink( @@ -493,7 +489,7 @@ impl ReclusterTableInterpreter { tbl: &Arc, push_downs: &mut Option, limit: Option, - ) -> Result> { + ) -> Result>> { let Some((parts, snapshot)) = tbl .recluster(self.ctx.clone(), push_downs.clone(), limit) .await? @@ -516,12 +512,13 @@ impl ReclusterTableInterpreter { removed_segment_indexes, removed_segment_summary, } => { - let root = PhysicalPlan::Recluster(Box::new(Recluster { + let root = Box::new(Recluster { tasks, - table_info: table_info.clone(), - plan_id: u32::MAX, table_meta_timestamps, - })); + + table_info: table_info.clone(), + meta: PhysicalPlanMeta::new("Recluster"), + }); Self::add_commit_sink( root, @@ -539,13 +536,13 @@ impl ReclusterTableInterpreter { } ReclusterParts::Compact(parts) => { let merge_meta = parts.partitions_type() == PartInfoType::LazyLevel; - let root = PhysicalPlan::CompactSource(Box::new(CompactSource { + let root = Box::new(CompactSource { parts, table_info: table_info.clone(), column_ids: snapshot.schema.to_leaf_column_id_set(), - plan_id: u32::MAX, table_meta_timestamps, - })); + meta: PhysicalPlanMeta::new("CompactSource"), + }); Self::add_commit_sink( root, @@ -704,43 +701,42 @@ impl ReclusterTableInterpreter { } fn add_commit_sink( - input: PhysicalPlan, + mut input: Box, is_distributed: bool, table_info: TableInfo, snapshot: Arc, merge_meta: bool, recluster_info: Option, table_meta_timestamps: TableMetaTimestamps, - ) -> PhysicalPlan { - let plan = if is_distributed { - PhysicalPlan::Exchange(Exchange { - plan_id: 0, - input: Box::new(input), + ) -> Box { + if is_distributed { + input = Box::new(Exchange { + input, kind: FragmentKind::Merge, keys: vec![], allow_adjust_parallelism: true, ignore_exchange: false, - }) - } else { - input - }; + meta: PhysicalPlanMeta::new("Exchange"), + }); + } - let kind = if recluster_info.is_some() { - MutationKind::Recluster - } else { - MutationKind::Compact - }; - PhysicalPlan::CommitSink(Box::new(CommitSink { - input: Box::new(plan), + let mut kind = MutationKind::Compact; + + if recluster_info.is_some() { + kind = MutationKind::Recluster + } + + Box::new(CommitSink { + input, table_info, snapshot: Some(snapshot), commit_type: CommitType::Mutation { kind, merge_meta }, update_stream_meta: vec![], deduplicated_label: None, table_meta_timestamps, - plan_id: u32::MAX, recluster_info, - })) + meta: PhysicalPlanMeta::new("CommitSink"), + }) } } diff --git a/src/query/service/src/lib.rs b/src/query/service/src/lib.rs index a6115a346ab93..8f555b64cb849 100644 --- a/src/query/service/src/lib.rs +++ b/src/query/service/src/lib.rs @@ -37,6 +37,7 @@ #![feature(variant_count)] #![feature(duration_constructors)] #![feature(get_mut_unchecked)] +#![feature(box_into_inner)] #![allow(clippy::diverging_sub_expression)] #![allow(clippy::arc_with_non_send_sync)] diff --git a/src/query/service/src/pipelines/pipeline_builder.rs b/src/query/service/src/pipelines/pipeline_builder.rs index e2e2b1c30a001..e0210b0653eef 100644 --- a/src/query/service/src/pipelines/pipeline_builder.rs +++ b/src/query/service/src/pipelines/pipeline_builder.rs @@ -26,7 +26,7 @@ use databend_common_pipeline_core::processors::PlanScopeGuard; use databend_common_pipeline_core::ExecutionInfo; use databend_common_pipeline_core::Pipeline; use databend_common_settings::Settings; -use databend_common_sql::executor::PhysicalPlan; +use databend_common_sql::executor::{IPhysicalPlan, PhysicalPlan}; use super::PipelineBuilderData; use crate::interpreters::CreateTableInterpreter; @@ -81,8 +81,8 @@ impl PipelineBuilder { } } - pub fn finalize(mut self, plan: &PhysicalPlan) -> Result { - self.build_pipeline(plan)?; + pub fn finalize(mut self, plan: &Box) -> Result { + // self.build_pipeline(plan)?; for source_pipeline in &self.pipelines { if !source_pipeline.is_complete_pipeline()? { @@ -136,7 +136,7 @@ impl PipelineBuilder { let scope = PlanScope::create( plan.get_id(), - plan.name(), + plan.get_name(), Arc::new(desc), Arc::new(profile_labels), ); diff --git a/src/query/service/src/pipelines/processors/transforms/transform_recursive_cte_source.rs b/src/query/service/src/pipelines/processors/transforms/transform_recursive_cte_source.rs index 41469da10a4f4..4283a4b5f8b58 100644 --- a/src/query/service/src/pipelines/processors/transforms/transform_recursive_cte_source.rs +++ b/src/query/service/src/pipelines/processors/transforms/transform_recursive_cte_source.rs @@ -314,8 +314,7 @@ async fn create_memory_table_for_cte_scan( table_indexes: None, attached_columns: None, }; - let create_table_interpreter = - CreateTableInterpreter::try_create(ctx.clone(), create_table_plan)?; + let create_table_interpreter = CreateTableInterpreter::try_create(ctx.clone(), create_table_plan)?; let _ = create_table_interpreter.execute(ctx.clone()).await?; } PhysicalPlan::Shuffle(plan) => { diff --git a/src/query/service/src/schedulers/fragments/fragmenter.rs b/src/query/service/src/schedulers/fragments/fragmenter.rs index dbc9872517533..bb1fe33df58df 100644 --- a/src/query/service/src/schedulers/fragments/fragmenter.rs +++ b/src/query/service/src/schedulers/fragments/fragmenter.rs @@ -12,12 +12,14 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::collections::HashMap; use std::sync::Arc; use databend_common_catalog::table_context::TableContext; use databend_common_exception::Result; +use databend_common_functions::aggregates::assert_arguments; use databend_common_meta_types::NodeInfo; -use databend_common_sql::executor::physical_plans::CompactSource; +use databend_common_sql::executor::physical_plans::{BroadcastSink, CompactSource}; use databend_common_sql::executor::physical_plans::ConstantTableScan; use databend_common_sql::executor::physical_plans::CopyIntoTable; use databend_common_sql::executor::physical_plans::CopyIntoTableSource; @@ -31,7 +33,7 @@ use databend_common_sql::executor::physical_plans::Recluster; use databend_common_sql::executor::physical_plans::ReplaceInto; use databend_common_sql::executor::physical_plans::TableScan; use databend_common_sql::executor::physical_plans::UnionAll; -use databend_common_sql::executor::DeriveHandle; +use databend_common_sql::executor::{DeriveHandle, PhysicalPlanDynExt, PhysicalPlanMeta, PhysicalPlanVisitor}; use databend_common_sql::executor::IPhysicalPlan; use crate::clusters::ClusterHelper; @@ -122,66 +124,163 @@ impl Fragmenter { } } - pub fn build_fragment(mut self, plan: &Box) -> Result { - let root = plan.derive_with(&mut self); - let fragment_type = match plan { - PhysicalPlan::BroadcastSink(_) => FragmentType::Intermediate, - _ => FragmentType::Root, - }; - let mut root_fragment = PlanFragment { + pub fn build_fragment(mut self, plan: &Box) -> Result> { + let mut fragments = HashMap::new(); + let mut handle = FragmentDeriveHandle::new(self.query_id.clone(), self.ctx.clone(), &mut fragments); + let root = plan.derive_with(&mut handle); + + let mut fragment_type = FragmentType::Root; + if let Some(_broadcast_sink) = plan.downcast_ref::() { + fragment_type = FragmentType::Intermediate; + } + + fragments.insert(self.ctx.get_fragment_id(), PlanFragment { plan: root, fragment_type, fragment_id: self.ctx.get_fragment_id(), exchange: None, query_id: self.query_id.clone(), source_fragments: self.fragments, - }; - Self::resolve_fragment_connection(&mut root_fragment); + }); + + let edges = Self::collect_fragments_edge(fragments.values()); + + for (source, target) in edges { + if let Some(exchange_sink) = fragments[&source].plan.downcast_mut_ref::() { + exchange_sink.destination_fragment_id = target; + } + } - Ok(root_fragment) + Ok(fragments.into_values().collect::>()) } - fn resolve_fragment_connection(fragment: &mut PlanFragment) { - for source_fragment in fragment.source_fragments.iter_mut() { - if let PhysicalPlan::ExchangeSink(ExchangeSink { - destination_fragment_id, - .. - }) = &mut source_fragment.plan - { - // Fill the destination_fragment_id with parent fragment id. - *destination_fragment_id = fragment.fragment_id; + fn collect_fragments_edge<'a>(iter: impl Iterator) -> HashMap { + struct EdgeVisitor { + target_fragment_id: usize, + map: HashMap, + } + + impl PhysicalPlanVisitor for EdgeVisitor { + fn visit(&mut self, plan: &Box) -> Result<()> { + if let Some(v) = plan.downcast_ref::() { + if let Some(v) = self.map.insert(v.source_fragment_id, self.target_fragment_id) { + assert_eq!(v, self.target_fragment_id); + } + } + + Ok(()) } } + + let mut edges = HashMap::new(); + for fragment in iter { + let mut visitor = Box::new(EdgeVisitor { map: HashMap::new(), target_fragment_id: fragment.fragment_id }); + fragment.plan.visit(&mut visitor).unwrap(); + edges.extend(visitor.map.into_iter()) + } + + edges } } -impl DeriveHandle for Fragmenter { +struct FragmentDeriveHandle<'a> { + state: State, + query_id: String, + ctx: Arc, + fragments: &'a mut HashMap, +} + +impl<'a> FragmentDeriveHandle<'a> { + pub fn new(query_id: String, ctx: Arc, fragments: &'a mut HashMap) -> Box { + Box::new(FragmentDeriveHandle { + ctx, + query_id, + fragments, + state: State::Other, + }) + } +} + +impl<'a> DeriveHandle for FragmentDeriveHandle<'a> { fn derive( &mut self, v: &Box, children: Vec>, ) -> std::result::Result, Vec>> { - let Some(compact_source) = v.downcast_ref::() else { - return Err(children); - }; + if let Some(_recluster) = v.downcast_ref::() { + self.state = State::Recluster; + } else if let Some(_table_scan) = v.downcast_ref::() { + self.state = State::SelectLeaf; + } else if let Some(_const_table_scan) = v.downcast_ref::() { + self.state = State::SelectLeaf; + } else if let Some(_compact_source) = v.downcast_ref::() { + self.state = State::Compact; + } else if let Some(_replace_into) = v.downcast_ref::() { + self.state = State::ReplaceInto; + } else if let Some(_mutation_source) = v.downcast_ref::() { + self.state = State::MutationSource; + } + + if let Some(exchange) = v.downcast_ref::() { + let input = children.remove(0); + let input_schema = input.output_schema()?; + + let plan_id = v.get_id(); + let source_fragment_id = self.ctx.get_fragment_id(); + + let plan = Box::new(ExchangeSink { + input, + schema: input_schema.clone(), + kind: exchange.kind.clone(), + keys: exchange.keys.clone(), + + query_id: self.query_id.clone(), + + // We will connect the fragments later, so we just + // set the fragment id to a invalid value here. + destination_fragment_id: usize::MAX, + ignore_exchange: exchange.ignore_exchange, + allow_adjust_parallelism: exchange.allow_adjust_parallelism, + meta: PhysicalPlanMeta::with_plan_id("ExchangeSink", plan_id), + }); + + let fragment_type = match self.state { + State::SelectLeaf => FragmentType::Source, + State::MutationSource => FragmentType::MutationSource, + State::Other => FragmentType::Intermediate, + State::ReplaceInto => FragmentType::ReplaceInto, + State::Compact => FragmentType::Compact, + State::Recluster => FragmentType::Recluster, + }; + + self.state = State::Other; + let exchange = Self::get_exchange(self.ctx.clone(), &plan)?; + + let source_fragment = PlanFragment { + plan, + exchange, + fragment_type, + source_fragments: vec![], + fragment_id: source_fragment_id, + query_id: self.query_id.clone(), + }; + + self.fragments.insert(source_fragment_id, source_fragment); + + return Ok(Box::new(ExchangeSource { + schema: input_schema, + query_id: self.query_id.clone(), + + source_fragment_id, + meta: PhysicalPlanMeta::with_plan_id("ExchangeSource", plan_id), + })); + } + + Err(children) } } // impl PhysicalPlanReplacer for Fragmenter { -// fn replace_recluster(&mut self, plan: &Recluster) -> Result { -// self.state = State::Recluster; -// Ok(PhysicalPlan::Recluster(Box::new(plan.clone()))) -// } -// -// fn replace_table_scan(&mut self, plan: &TableScan) -> Result { -// self.state = State::SelectLeaf; -// Ok(PhysicalPlan::TableScan(plan.clone())) -// } -// -// fn replace_constant_table_scan(&mut self, plan: &ConstantTableScan) -> Result { -// self.state = State::SelectLeaf; -// Ok(PhysicalPlan::ConstantTableScan(plan.clone())) -// } -// + // fn replace_hash_join(&mut self, plan: &HashJoin) -> Result { // let mut fragments = vec![]; // let build_input = self.replace(plan.build.as_ref())?; @@ -218,69 +317,6 @@ impl DeriveHandle for Fragmenter { // })) // } // -// fn replace_exchange(&mut self, plan: &Exchange) -> Result { -// // Recursively rewrite input -// let input = self.replace(plan.input.as_ref())?; -// let input_schema = input.output_schema()?; -// -// let plan_id = plan.plan_id; -// -// let source_fragment_id = self.ctx.get_fragment_id(); -// let plan = PhysicalPlan::ExchangeSink(ExchangeSink { -// // TODO(leiysky): we reuse the plan id here, -// // should generate a new one for the sink. -// plan_id, -// -// input: Box::new(input), -// schema: input_schema.clone(), -// kind: plan.kind.clone(), -// keys: plan.keys.clone(), -// -// query_id: self.query_id.clone(), -// -// // We will connect the fragments later, so we just -// // set the fragment id to a invalid value here. -// destination_fragment_id: usize::MAX, -// ignore_exchange: plan.ignore_exchange, -// allow_adjust_parallelism: plan.allow_adjust_parallelism, -// }); -// let fragment_type = match self.state { -// State::SelectLeaf => FragmentType::Source, -// State::MutationSource => FragmentType::MutationSource, -// State::Other => FragmentType::Intermediate, -// State::ReplaceInto => FragmentType::ReplaceInto, -// State::Compact => FragmentType::Compact, -// State::Recluster => FragmentType::Recluster, -// }; -// self.state = State::Other; -// let exchange = Self::get_exchange(self.ctx.clone(), &plan)?; -// -// let mut source_fragment = PlanFragment { -// plan, -// fragment_type, -// fragment_id: source_fragment_id, -// exchange, -// query_id: self.query_id.clone(), -// source_fragments: std::mem::take(&self.fragments), -// }; -// -// // Fill the destination_fragment_id for source fragments of `source_fragment`. -// Self::resolve_fragment_connection(&mut source_fragment); -// -// self.fragments.push(source_fragment); -// -// Ok(PhysicalPlan::ExchangeSource(ExchangeSource { -// // TODO(leiysky): we reuse the plan id here, -// // should generate a new one for the source. -// plan_id, -// -// schema: input_schema, -// query_id: self.query_id.clone(), -// -// source_fragment_id, -// })) -// } -// // fn replace_union(&mut self, plan: &UnionAll) -> Result { // let mut fragments = vec![]; // let left_input = self.replace(plan.left.as_ref())?; @@ -307,48 +343,4 @@ impl DeriveHandle for Fragmenter { // ..plan.clone() // })) // } -// -// // TODO(Sky): remove redundant code -// fn replace_copy_into_table(&mut self, plan: &CopyIntoTable) -> Result { -// match &plan.source { -// CopyIntoTableSource::Stage(_) => { -// self.state = State::SelectLeaf; -// Ok(PhysicalPlan::CopyIntoTable(Box::new(plan.clone()))) -// } -// CopyIntoTableSource::Query(query_physical_plan) => { -// let input = self.replace(query_physical_plan)?; -// Ok(PhysicalPlan::CopyIntoTable(Box::new(CopyIntoTable { -// source: CopyIntoTableSource::Query(Box::new(input)), -// ..plan.clone() -// }))) -// } -// } -// } -// -// fn replace_compact_source(&mut self, plan: &CompactSource) -> Result { -// self.state = State::Compact; -// Ok(PhysicalPlan::CompactSource(Box::new(plan.clone()))) -// } -// -// fn replace_replace_into(&mut self, plan: &ReplaceInto) -> Result { -// let input = self.replace(&plan.input)?; -// self.state = State::ReplaceInto; -// Ok(PhysicalPlan::ReplaceInto(Box::new(ReplaceInto { -// input: Box::new(input), -// ..plan.clone() -// }))) -// } -// -// fn replace_mutation_source(&mut self, plan: &MutationSource) -> Result { -// self.state = State::MutationSource; -// Ok(PhysicalPlan::MutationSource(plan.clone())) -// } -// -// fn replace_mutation(&mut self, plan: &Mutation) -> Result { -// let input = self.replace(&plan.input)?; -// Ok(PhysicalPlan::Mutation(Box::new(Mutation { -// input: Box::new(input), -// ..plan.clone() -// }))) -// } // } diff --git a/src/query/service/src/schedulers/fragments/plan_fragment.rs b/src/query/service/src/schedulers/fragments/plan_fragment.rs index 79480c6dcb1d5..18466cad61370 100644 --- a/src/query/service/src/schedulers/fragments/plan_fragment.rs +++ b/src/query/service/src/schedulers/fragments/plan_fragment.rs @@ -492,26 +492,22 @@ impl DeriveHandle for ReadSourceDeriveHandle { children: Vec>, ) -> std::result::Result, Vec>> { if let Some(table_scan) = v.downcast_ref::() { - let source = self.sources.remove(&table_scan.get_id()).ok_or_else(|| { - ErrorCode::Internal(format!( - "Cannot find data source for table scan plan {}", - table_scan.get_id() - )) - })?; + let Some(source) = self.sources.remove(&table_scan.get_id()) else { + unreachable!("Cannot find data source for table scan plan {}", table_scan.get_id()) + }; return Ok(Box::new(TableScan { source: Box::new(DataSourcePlan::try_from(source)?), ..table_scan.clone() })); } else if let Some(table_scan) = v.downcast_ref::() { - let source = self.sources.remove(&table_scan.get_id()).ok_or_else(|| { - ErrorCode::Internal(format!( - "Cannot find data source for constant table scan plan {}", - table_scan.get_id() - )) - })?; + let Some(source) = self.sources.remove(&table_scan.get_id()) else { + unreachable!("Cannot find data source for constant table scan plan {}", table_scan.get_id()) + }; - let const_table_columns = ConstTableColumn::try_from(source)?; + let Ok(const_table_columns) = ConstTableColumn::try_from(source) else { + unreachable!("Cannot convert Table to Vec") + }; return Ok(Box::new(ConstantTableScan { values: const_table_columns.columns, @@ -651,6 +647,6 @@ impl DeriveHandle for ReplaceDeriveHandle { })); } - None + Err(children) } } diff --git a/src/query/service/src/schedulers/fragments/query_fragment_actions.rs b/src/query/service/src/schedulers/fragments/query_fragment_actions.rs index 46ce59b5568c0..6a5145d251129 100644 --- a/src/query/service/src/schedulers/fragments/query_fragment_actions.rs +++ b/src/query/service/src/schedulers/fragments/query_fragment_actions.rs @@ -24,8 +24,8 @@ use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_expression::DataSchemaRef; use databend_common_meta_types::NodeInfo; -use databend_common_sql::executor::IPhysicalPlan; - +use databend_common_sql::executor::{IPhysicalPlan, PhysicalPlanDynExt}; +use databend_common_sql::executor::physical_plans::ExchangeSink; use crate::clusters::ClusterHelper; use crate::servers::flight::v1::exchange::DataExchange; use crate::servers::flight::v1::packets::DataflowDiagramBuilder; @@ -132,7 +132,7 @@ impl QueryFragmentsActions { let mut fragment_ids = Vec::new(); for fragment_actions in &self.fragments_actions { let plan = &fragment_actions.fragment_actions[0].physical_plan; - if !matches!(plan, PhysicalPlan::ExchangeSink(_)) { + if plan.downcast_ref::().is_none() { fragment_ids.push(fragment_actions.fragment_id); } } diff --git a/src/query/service/src/schedulers/fragments/query_fragment_actions_display.rs b/src/query/service/src/schedulers/fragments/query_fragment_actions_display.rs index adb0b6c3bcd18..ef7127be3479f 100644 --- a/src/query/service/src/schedulers/fragments/query_fragment_actions_display.rs +++ b/src/query/service/src/schedulers/fragments/query_fragment_actions_display.rs @@ -14,7 +14,7 @@ use std::fmt::Display; use std::fmt::Formatter; - +use databend_common_sql::executor::PhysicalPlanDynExt; use databend_common_sql::MetadataRef; use crate::schedulers::QueryFragmentActions; @@ -77,11 +77,14 @@ impl Display for QueryFragmentActionsWrap<'_> { if !self.inner.fragment_actions.is_empty() { let fragment_action = &self.inner.fragment_actions[0]; - let plan_display_string = fragment_action - .physical_plan - .format(self.metadata.clone(), Default::default()) - .and_then(|node| Ok(node.format_pretty_with_prefix(" ")?)) - .unwrap(); + let plan_display_string = { + let metadata = self.metadata.read(); + fragment_action + .physical_plan + .format(&metadata, Default::default()) + .and_then(|node| Ok(node.format_pretty_with_prefix(" ")?)) + .unwrap() + }; write!(f, "{}", plan_display_string)?; } diff --git a/src/query/service/src/schedulers/scheduler.rs b/src/query/service/src/schedulers/scheduler.rs index 257737340ea10..2d0ddeed40ec6 100644 --- a/src/query/service/src/schedulers/scheduler.rs +++ b/src/query/service/src/schedulers/scheduler.rs @@ -43,7 +43,7 @@ use crate::stream::PullingExecutorStream; pub async fn build_query_pipeline( ctx: &Arc, result_columns: &[ColumnBinding], - plan: &PhysicalPlan, + plan: &Box, ignore_result: bool, ) -> Result { let mut build_res = build_query_pipeline_without_render_result_set(ctx, plan).await?; @@ -84,11 +84,12 @@ pub async fn build_local_pipeline( ) -> Result { let pipeline = PipelineBuilder::create(ctx.get_function_context()?, ctx.get_settings(), ctx.clone()); - let mut build_res = pipeline.finalize(plan)?; + // let mut build_res = pipeline.finalize(plan)?; - let settings = ctx.get_settings(); - build_res.set_max_threads(settings.get_max_threads()? as usize); - Ok(build_res) + // let settings = ctx.get_settings(); + // build_res.set_max_threads(settings.get_max_threads()? as usize); + // Ok(build_res) + unimplemented!() } /// Build distributed pipeline via fragment and actions. diff --git a/src/query/service/src/servers/flight/v1/exchange/exchange_manager.rs b/src/query/service/src/servers/flight/v1/exchange/exchange_manager.rs index 1bb5af570df33..60a4ab64aefb3 100644 --- a/src/query/service/src/servers/flight/v1/exchange/exchange_manager.rs +++ b/src/query/service/src/servers/flight/v1/exchange/exchange_manager.rs @@ -36,7 +36,7 @@ use databend_common_exception::Result; use databend_common_grpc::ConnectionFactory; use databend_common_pipeline_core::basic_callback; use databend_common_pipeline_core::ExecutionInfo; -use databend_common_sql::executor::PhysicalPlan; +use databend_common_sql::executor::{IPhysicalPlan, PhysicalPlan}; use fastrace::prelude::*; use log::warn; use parking_lot::Mutex; @@ -964,7 +964,7 @@ impl QueryCoordinator { struct FragmentCoordinator { initialized: bool, fragment_id: usize, - physical_plan: PhysicalPlan, + physical_plan: Box, data_exchange: Option, pipeline_build_res: Option, } diff --git a/src/query/service/src/servers/flight/v1/packets/packet_fragment.rs b/src/query/service/src/servers/flight/v1/packets/packet_fragment.rs index e3cee46023264..4b722bda6e844 100644 --- a/src/query/service/src/servers/flight/v1/packets/packet_fragment.rs +++ b/src/query/service/src/servers/flight/v1/packets/packet_fragment.rs @@ -23,8 +23,8 @@ use crate::sql::executor::PhysicalPlan; #[derive(Clone, serde::Serialize, serde::Deserialize)] pub struct QueryFragment { pub fragment_id: usize, - pub physical_plan: Box, pub data_exchange: Option, + pub physical_plan: Box, } impl QueryFragment { diff --git a/src/query/service/tests/it/pipelines/builders/runtime_filter.rs b/src/query/service/tests/it/pipelines/builders/runtime_filter.rs index b739612ea5a44..a7cb04f5c0a69 100644 --- a/src/query/service/tests/it/pipelines/builders/runtime_filter.rs +++ b/src/query/service/tests/it/pipelines/builders/runtime_filter.rs @@ -18,7 +18,7 @@ use databend_common_base::base::tokio; use databend_common_exception::Result; use databend_common_expression::SendableDataBlockStream; use databend_common_sql::executor::physical_plans::HashJoin; -use databend_common_sql::executor::PhysicalPlan; +use databend_common_sql::executor::{IPhysicalPlan, PhysicalPlan}; use databend_common_sql::executor::PhysicalPlanBuilder; use databend_common_sql::plans::Plan; use databend_common_sql::Planner; @@ -42,7 +42,7 @@ async fn execute_sql(ctx: Arc, sql: &str) -> Result, sql: &str) -> Result { +async fn physical_plan(ctx: Arc, sql: &str) -> Result> { let plan = plan_sql(ctx.clone(), sql).await?; match plan { Plan::Query { diff --git a/src/query/sql/src/executor/mod.rs b/src/query/sql/src/executor/mod.rs index 3782dee62da8d..f25007281bc7f 100644 --- a/src/query/sql/src/executor/mod.rs +++ b/src/query/sql/src/executor/mod.rs @@ -32,4 +32,5 @@ pub use physical_plan_builder::PhysicalPlanBuilder; pub use physical_plans::build_broadcast_plans; pub use physical_plans::PhysicalRuntimeFilter; pub use physical_plans::PhysicalRuntimeFilters; +pub use physical_plan::PhysicalPlanVisitor; pub use util::*; diff --git a/src/query/sql/src/executor/physical_plan.rs b/src/query/sql/src/executor/physical_plan.rs index 4dc78e202dc32..bd1ce0d9e11fd 100644 --- a/src/query/sql/src/executor/physical_plan.rs +++ b/src/query/sql/src/executor/physical_plan.rs @@ -23,7 +23,7 @@ use databend_common_catalog::plan::DataSourceInfo; use databend_common_catalog::plan::DataSourcePlan; use databend_common_catalog::plan::PartStatistics; use databend_common_catalog::plan::PartitionsShuffleKind; -use databend_common_exception::Result; +use databend_common_exception::{ErrorCode, Result}; use databend_common_expression::DataSchemaRef; use databend_common_functions::BUILTIN_FUNCTIONS; use databend_common_pipeline_core::PlanProfile; @@ -95,14 +95,18 @@ pub struct PhysicalPlanMeta { impl PhysicalPlanMeta { pub fn new(name: impl Into) -> PhysicalPlanMeta { + PhysicalPlanMeta::with_plan_id(name, 0) + } + + pub fn with_plan_id(name: impl Into, plan_id: u32) -> PhysicalPlanMeta { PhysicalPlanMeta { - plan_id: 0, + plan_id, name: name.into(), } } } -pub trait DeriveHandle { +pub trait DeriveHandle: Send + Sync + 'static { fn derive( &mut self, v: &Box, @@ -147,11 +151,11 @@ pub trait IPhysicalPlan: Debug + Send + Sync + 'static { } } - fn children(&self) -> Box> + '_> { + fn children(&self) -> Box> + '_> { Box::new(std::iter::empty()) } - fn children_mut(&mut self) -> Box> + '_> { + fn children_mut(&mut self) -> Box> + '_> { Box::new(std::iter::empty()) } @@ -215,14 +219,8 @@ pub trait IPhysicalPlan: Debug + Send + Sync + 'static { fn derive(&self, children: Vec>) -> Box; } -pub trait PhysicalPlanExt { - fn clone_box(&self) -> Box; -} - -impl PhysicalPlanExt for T { - fn clone_box(&self) -> Box { - Box::new(self.clone()) - } +pub trait PhysicalPlanVisitor { + fn visit(&mut self, plan: &Box) -> Result<()>; } pub trait PhysicalPlanDynExt { @@ -250,6 +248,8 @@ pub trait PhysicalPlanDynExt { fn downcast_mut_ref(&mut self) -> Option<&mut To>; fn derive_with(&self, handle: &mut Box) -> Box; + + fn visit(&self, visitor: &mut Box) -> Result<()>; } impl PhysicalPlanDynExt for Box { @@ -305,24 +305,24 @@ impl PhysicalPlanDynExt for Box { self.derive(children) } -} -struct DummyDeriveHandle; + fn visit(&self, visitor: &mut Box) -> Result<()> { + for child in self.children() { + child.visit(visitor)?; + } -impl DeriveHandle for DummyDeriveHandle { - fn derive( - &mut self, - _: &Box, - children: Vec>, - ) -> std::result::Result, Vec>> { - Err(children) + visitor.visit(self) } } impl Clone for Box { fn clone(&self) -> Self { - let mut handle: Box = Box::new(DummyDeriveHandle); - self.derive_with(&mut handle) + let mut children = vec![]; + for child in self.children() { + children.push(child.clone()); + } + + self.derive(children) } } diff --git a/src/query/sql/src/executor/physical_plan_visitor.rs b/src/query/sql/src/executor/physical_plan_visitor.rs index 76d4e2944fd7f..987f4b4c2e1e6 100644 --- a/src/query/sql/src/executor/physical_plan_visitor.rs +++ b/src/query/sql/src/executor/physical_plan_visitor.rs @@ -12,65 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -use databend_common_exception::Result; - -use super::physical_plans::AddStreamColumn; -use super::physical_plans::BroadcastSink; -use super::physical_plans::BroadcastSource; -use super::physical_plans::CacheScan; -use super::physical_plans::ExpressionScan; -use super::physical_plans::HilbertPartition; -use super::physical_plans::MutationManipulate; -use super::physical_plans::MutationOrganize; -use super::physical_plans::MutationSplit; -use super::physical_plans::RecursiveCteScan; use crate::executor::physical_plan::PhysicalPlan; -use crate::executor::physical_plan::PhysicalPlanExt; -use crate::executor::physical_plans::AggregateExpand; -use crate::executor::physical_plans::AggregateFinal; -use crate::executor::physical_plans::AggregatePartial; -use crate::executor::physical_plans::AsyncFunction; -use crate::executor::physical_plans::ChunkAppendData; -use crate::executor::physical_plans::ChunkCastSchema; -use crate::executor::physical_plans::ChunkCommitInsert; -use crate::executor::physical_plans::ChunkEvalScalar; -use crate::executor::physical_plans::ChunkFillAndReorder; -use crate::executor::physical_plans::ChunkFilter; -use crate::executor::physical_plans::ChunkMerge; -use crate::executor::physical_plans::ColumnMutation; -use crate::executor::physical_plans::CommitSink; -use crate::executor::physical_plans::CompactSource; -use crate::executor::physical_plans::ConstantTableScan; -use crate::executor::physical_plans::CopyIntoLocation; -use crate::executor::physical_plans::CopyIntoTable; -use crate::executor::physical_plans::CopyIntoTableSource; -use crate::executor::physical_plans::DistributedInsertSelect; -use crate::executor::physical_plans::Duplicate; -use crate::executor::physical_plans::EvalScalar; -use crate::executor::physical_plans::Exchange; -use crate::executor::physical_plans::ExchangeSink; -use crate::executor::physical_plans::ExchangeSource; -use crate::executor::physical_plans::Filter; -use crate::executor::physical_plans::HashJoin; -use crate::executor::physical_plans::Limit; -use crate::executor::physical_plans::Mutation; -use crate::executor::physical_plans::MutationSource; -use crate::executor::physical_plans::ProjectSet; -use crate::executor::physical_plans::RangeJoin; -use crate::executor::physical_plans::Recluster; -use crate::executor::physical_plans::ReplaceAsyncSourcer; -use crate::executor::physical_plans::ReplaceDeduplicate; -use crate::executor::physical_plans::ReplaceInto; -use crate::executor::physical_plans::RowFetch; -use crate::executor::physical_plans::Shuffle; -use crate::executor::physical_plans::Sort; -use crate::executor::physical_plans::TableScan; -use crate::executor::physical_plans::Udf; -use crate::executor::physical_plans::UnionAll; -use crate::executor::physical_plans::Window; -use crate::executor::physical_plans::WindowPartition; use crate::executor::IPhysicalPlan; -use crate::executor::PhysicalPlanMeta; + // pub trait PhysicalPlanReplacer { // fn replace(&mut self, plan: &PhysicalPlan) -> Result { // match plan { diff --git a/src/query/sql/src/executor/physical_plans/physical_table_scan.rs b/src/query/sql/src/executor/physical_plans/physical_table_scan.rs index 6771a38cdfa6c..72fbf9bc2a1cc 100644 --- a/src/query/sql/src/executor/physical_plans/physical_table_scan.rs +++ b/src/query/sql/src/executor/physical_plans/physical_table_scan.rs @@ -87,7 +87,7 @@ use crate::DUMMY_TABLE_INDEX; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct TableScan { - meta: PhysicalPlanMeta, + pub meta: PhysicalPlanMeta, pub scan_id: usize, pub name_mapping: BTreeMap, pub source: Box,