Skip to content

Commit d6d2159

Browse files
authored
refactor: Reduce parallelism when vacuuming dropped tables (#16312)
refactor: Reduce parallelism when vacuuming dropped tables
1 parent 66c5f2e commit d6d2159

File tree

1 file changed

+11
-1
lines changed

1 file changed

+11
-1
lines changed

src/query/ee/src/storages/fuse/operations/vacuum_drop_tables.rs

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -103,10 +103,20 @@ pub async fn vacuum_drop_tables_by_table_info(
103103
) -> Result<Option<Vec<VacuumDropFileInfo>>> {
104104
let start = Instant::now();
105105
let num_tables = table_infos.len();
106-
info!("vacuum dropped tables, number of tables: {}", num_tables);
106+
107+
// - for each vacuum task, the tables passed to it will be processed sequentially
108+
// - while removing one table's data, at most 1000 objects will be deleted (in batch)
109+
// - let's assume that the rate limit is 3500 (individual) objects per second:
110+
// A parallelism degree of up to 3 appears to be safe.
111+
let num_threads = std::cmp::min(num_threads, 3);
107112

108113
let batch_size = (num_tables / num_threads).clamp(1, 50);
109114

115+
info!(
116+
"vacuum dropped tables, number of tables: {}, batch_size: {}, parallelism degree: {}",
117+
num_tables, batch_size, num_threads
118+
);
119+
110120
let result = if batch_size >= table_infos.len() {
111121
do_vacuum_drop_table(table_infos, dry_run_limit).await?
112122
} else {

0 commit comments

Comments
 (0)