Skip to content

Commit 3434c5e

Browse files
authored
fix(query): attach table should not collect statistics in system.columns (#18310)
**Solution:** 1. **Disable Column Statistics Collection for Attached Tables by Default:** To improve `system.columns` query performance and prevent collecting statistics based on potentially inaccurate snapshots, we will, by default, disable column statistics collection for attached tables. 2. **Introduce New Setting `enable_collect_column_statistics`:** This new setting provides granular control over column statistics collection behavior. * **Default Value:** `1` (enables statistics collection, but attached tables remain subject to the aforementioned restriction). * **Disable:** Setting it to `0` will completely stop column statistics collection for all tables within `system.columns` (including non-attached tables), further boosting performance. * **Configuration:** `SET GLOBAL enable_collect_column_statistics = [0|1];`
1 parent 316bafb commit 3434c5e

File tree

4 files changed

+42
-3
lines changed

4 files changed

+42
-3
lines changed

src/query/settings/src/settings_default.rs

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -960,6 +960,13 @@ impl DefaultSettings {
960960
scope: SettingScope::Both,
961961
range: Some(SettingRange::Numeric(0..=1)),
962962
}),
963+
("enable_collect_column_statistics", DefaultSettingValue {
964+
value: UserSettingValue::UInt64(1),
965+
desc: "Collect column statistic in system.columns(enable by default).",
966+
mode: SettingMode::Both,
967+
scope: SettingScope::Both,
968+
range: Some(SettingRange::Numeric(0..=1)),
969+
}),
963970
("enable_expand_roles", DefaultSettingValue {
964971
value: UserSettingValue::UInt64(1),
965972
desc: "Enable expand roles when execute show grants statement(enable by default).",

src/query/settings/src/settings_getter_setter.rs

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -548,6 +548,10 @@ impl Settings {
548548
Ok(self.try_get_u64("enable_experimental_rbac_check")? != 0)
549549
}
550550

551+
pub fn get_enable_collect_column_statistics(&self) -> Result<bool> {
552+
Ok(self.try_get_u64("enable_collect_column_statistics")? != 0)
553+
}
554+
551555
pub fn get_enable_expand_roles(&self) -> Result<bool> {
552556
Ok(self.try_get_u64("enable_expand_roles")? != 0)
553557
}

src/query/storages/system/src/columns_table.rs

Lines changed: 29 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ use databend_common_catalog::catalog::CatalogManager;
1919
use databend_common_catalog::catalog_kind::CATALOG_DEFAULT;
2020
use databend_common_catalog::database::Database;
2121
use databend_common_catalog::plan::PushDownInfo;
22+
use databend_common_catalog::table::DummyColumnStatisticsProvider;
2223
use databend_common_catalog::table::Table;
2324
use databend_common_catalog::table_context::TableContext;
2425
use databend_common_exception::Result;
@@ -40,6 +41,7 @@ use databend_common_meta_app::schema::TableInfo;
4041
use databend_common_meta_app::schema::TableMeta;
4142
use databend_common_meta_app::tenant::Tenant;
4243
use databend_common_sql::Planner;
44+
use databend_common_storages_fuse::FuseTable;
4345
use databend_common_storages_stream::stream_table::StreamTable;
4446
use databend_common_storages_stream::stream_table::STREAM_ENGINE;
4547
use databend_common_storages_view::view_table::QUERY;
@@ -271,10 +273,7 @@ impl ColumnsTable {
271273
_ => {
272274
let schema = table.schema();
273275
let field_comments = table.field_comments();
274-
let columns_statistics =
275-
table.column_statistics_provider(ctx.clone()).await?;
276276

277-
let row_count = columns_statistics.num_rows();
278277
for (idx, field) in schema.fields().iter().enumerate() {
279278
let comment = if field_comments.len() == schema.fields.len()
280279
&& !field_comments[idx].is_empty()
@@ -285,6 +284,33 @@ impl ColumnsTable {
285284
"".to_string()
286285
};
287286

287+
// attach table should not collect statistics, source table column already collect them.
288+
let columns_statistics = if ctx
289+
.get_settings()
290+
.get_enable_collect_column_statistics()?
291+
&& !FuseTable::is_table_attached(
292+
&table.get_table_info().meta.options,
293+
) {
294+
table
295+
.column_statistics_provider(ctx.clone())
296+
.await
297+
.unwrap_or_else(|e| {
298+
let msg = format!(
299+
"Collect {}.{}.{} column statistics with error: {}",
300+
catalog.name(),
301+
database,
302+
table.name(),
303+
e
304+
);
305+
warn!("{}", msg);
306+
ctx.push_warning(msg);
307+
Box::new(DummyColumnStatisticsProvider)
308+
})
309+
} else {
310+
Box::new(DummyColumnStatisticsProvider)
311+
};
312+
313+
let row_count = columns_statistics.num_rows();
288314
let column_statistics =
289315
columns_statistics.column_statistics(field.column_id);
290316
rows.push(TableColumnInfo {

tests/suites/5_ee/04_attach_read_only/02_0004_attach_table.sh

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,8 @@ storage_prefix=$(mysql -uroot -h127.0.0.1 -P3307 -e "set global hide_options_in
3030

3131
comment "attaching table"
3232
echo "attach table table_to 's3://testbucket/admin/data/$storage_prefix' connection=(access_key_id ='minioadmin' secret_access_key ='minioadmin' endpoint_url='${STORAGE_S3_ENDPOINT_URL}');" | $BENDSQL_CLIENT_CONNECT
33+
# If failed will return err msg. Expect it success.
34+
echo "select * from system.columns where table='table_to' ignore_result;" | $BENDSQL_CLIENT_CONNECT
3335
echo "attach table table_to2 's3://testbucket/admin/data/$storage_prefix' connection=(connection_name ='my_conn')" | $BENDSQL_CLIENT_CONNECT
3436

3537

0 commit comments

Comments
 (0)