@@ -43,10 +43,10 @@ pub fn append2table(
43
43
ctx : Arc < QueryContext > ,
44
44
table : Arc < dyn Table > ,
45
45
source_schema : DataSchemaRef ,
46
- mut build_res : PipelineBuildResult ,
46
+ build_res : & mut PipelineBuildResult ,
47
+ overwrite : bool ,
47
48
) -> Result < ( ) > {
48
- let need_fill_missing_columns = table. schema ( ) != source_schema;
49
- if need_fill_missing_columns {
49
+ if table. schema ( ) != source_schema {
50
50
build_res
51
51
. main_pipeline
52
52
. add_transform ( |transform_input_port, transform_output_port| {
@@ -62,13 +62,36 @@ pub fn append2table(
62
62
63
63
table. append2 ( ctx. clone ( ) , & mut build_res. main_pipeline ) ?;
64
64
65
- let query_need_abort = ctx. query_need_abort ( ) ;
66
- let executor_settings = ExecutorSettings :: try_create ( & ctx. get_settings ( ) ) ?;
67
- build_res. set_max_threads ( ctx. get_settings ( ) . get_max_threads ( ) ? as usize ) ;
68
- let mut pipelines = build_res. sources_pipelines ;
69
- pipelines. push ( build_res. main_pipeline ) ;
70
- let executor = PipelineCompleteExecutor :: from_pipelines ( query_need_abort, pipelines, executor_settings) ?;
71
- executor. execute ( )
65
+ build_res. main_pipeline . set_on_finished ( move |may_error| {
66
+ // capture out variable
67
+ let overwrite = overwrite;
68
+ let ctx = ctx. clone ( ) ;
69
+ let table = table. clone ( ) ;
70
+
71
+ if may_error. is_none ( ) {
72
+ let append_entries = ctx. consume_precommit_blocks ( ) ;
73
+ // We must put the commit operation to global runtime, which will avoid the "dispatch dropped without returning error" in tower
74
+ let catalog_name = ctx. get_current_catalog ( ) ;
75
+ let commit_handle = GlobalIORuntime :: instance ( ) . spawn ( async move {
76
+ table
77
+ . commit_insertion ( ctx, & catalog_name, append_entries, overwrite)
78
+ . await
79
+ } ) ;
80
+
81
+ return match futures:: executor:: block_on ( commit_handle) {
82
+ Ok ( Ok ( _) ) => Ok ( ( ) ) ,
83
+ Ok ( Err ( error) ) => Err ( error) ,
84
+ Err ( cause) => Err ( ErrorCode :: PanicError ( format ! (
85
+ "Maybe panic while in commit insert. {}" ,
86
+ cause
87
+ ) ) )
88
+ } ;
89
+ }
90
+
91
+ Err ( may_error. as_ref ( ) . unwrap ( ) . clone ( ) )
92
+ } ) ;
93
+
94
+ Ok ( ( ) )
72
95
}
73
96
74
97
pub fn execute_pipeline ( ctx : Arc < QueryContext > , mut res : PipelineBuildResult ) -> Result < ( ) > {
@@ -81,6 +104,44 @@ pub fn execute_pipeline(ctx: Arc<QueryContext>, mut res: PipelineBuildResult) ->
81
104
executor. execute ( )
82
105
}
83
106
107
+ pub fn commit_table_pipeline (
108
+ ctx : Arc < QueryContext > ,
109
+ table : Arc < dyn Table > ,
110
+ overwrite : bool ,
111
+ build_res : & mut PipelineBuildResult ,
112
+ ) -> Result < ( ) > {
113
+ build_res. main_pipeline . set_on_finished ( move |may_error| {
114
+ // capture out variable
115
+ let overwrite = overwrite;
116
+ let ctx = ctx. clone ( ) ;
117
+ let table = table. clone ( ) ;
118
+
119
+ if may_error. is_none ( ) {
120
+ let append_entries = ctx. consume_precommit_blocks ( ) ;
121
+ // We must put the commit operation to global runtime, which will avoid the "dispatch dropped without returning error" in tower
122
+ let catalog_name = ctx. get_current_catalog ( ) ;
123
+ let commit_handle = GlobalIORuntime :: instance ( ) . spawn ( async move {
124
+ table
125
+ . commit_insertion ( ctx, & catalog_name, append_entries, overwrite)
126
+ . await
127
+ } ) ;
128
+
129
+ return match futures:: executor:: block_on ( commit_handle) {
130
+ Ok ( Ok ( _) ) => Ok ( ( ) ) ,
131
+ Ok ( Err ( error) ) => Err ( error) ,
132
+ Err ( cause) => Err ( ErrorCode :: PanicError ( format ! (
133
+ "Maybe panic while in commit insert. {}" ,
134
+ cause
135
+ ) ) )
136
+ } ;
137
+ }
138
+
139
+ Err ( may_error. as_ref ( ) . unwrap ( ) . clone ( ) )
140
+ } ) ;
141
+
142
+ Ok ( ( ) )
143
+ }
144
+
84
145
pub async fn commit2table (
85
146
ctx : Arc < QueryContext > ,
86
147
table : Arc < dyn Table > ,
0 commit comments