update doc, add more test case

Author: lichuang

docs/doc/14-sql-commands/00-ddl/20-table/60-optimize-table.md

@@ -5,10 +5,10 @@ title: OPTIMIZE TABLE
 The objective of optimizing a table in Databend is to compact or purge its historical data in your object storage. This helps save storage space and improve query efficiency.
 
 :::caution
-Databend's Time Travel feature relies on historical data. If you purge historical data from a table with the command `OPTIMIZE TABLE <your_table> PURGE` or `OPTIMIZE TABLE <your_table> ALL`, the table will not be eligible for time travel. The command removes all snapshots (except the most recent one) and their associated segments and block files.
+Databend's Time Travel feature relies on historical data. If you purge historical data from a table with the command `OPTIMIZE TABLE <your_table> PURGE` or `OPTIMIZE TABLE <your_table> ALL`, the table will not be eligible for time travel. The command removes all snapshots (except the most recent one) and their associated segments,block files and table statistic file.
 :::
 
-## What are Snapshot, Segment, and Block?
+## What are Snapshot, Segment, Block and Table statistic file?
 
 Snapshot, segment, and block are the concepts Databend uses for data storage. Databend uses them to construct a hierarchical structure for storing table data.
 
@@ -20,6 +20,8 @@ A snapshot is a JSON file that does not save the table's data but indicate the s
 
 A segment is a JSON file that organizes the storage blocks (at least 1, at most 1,000) where the data is stored. If you run [FUSE_SEGMENT](../../../15-sql-functions/111-system-functions/fuse_segment.md) against a snapshot with the snapshot ID, you can find which segments are referenced by the snapshot.
 
+A table statistic file is a JSON file that save table statistic data, such as distinct values of table column.
+
 Databends saves actual table data in parquet files and considers each parquet file as a block. If you run [FUSE_BLOCK](../../../15-sql-functions/111-system-functions/fuse_block.md) against a snapshot with the snapshot ID, you can find which blocks are referenced by the snapshot.
 
 Databend creates a unique ID for each database and table for storing the snapshot, segment, and block files and saves them to your object storage in the path `<bucket_name>/[root]/<db_id>/<table_id>/`. Each snapshot, segment, and block file is named with a UUID (32-character lowercase hexadecimal string).
@@ -29,6 +31,7 @@ Databend creates a unique ID for each database and table for storing the snapsho
 | Snapshot | JSON    | `<32bitUUID>_<version>.json`    | `<bucket_name>/[root]/<db_id>/<table_id>/_ss/`   |
 | Segment  | JSON    | `<32bitUUID>_<version>.json`    | `<bucket_name>/[root]/<db_id>/<table_id>/_sg/`   |
 | Block    | parquet | `<32bitUUID>_<version>.parquet` | `<bucket_name>/[root]/<db_id>/<table_id>/_b/` |
+| Table statistic | JSON    | `<32bitUUID>_<version>.json`    | `<bucket_name>/[root]/<db_id>/<table_id>/_ts/`   |
 
 ## Table Optimization Considerations
 
@@ -69,7 +72,7 @@ OPTIMIZE TABLE [database.]table_name [ PURGE | COMPACT | ALL | STATISTIC ] [SEGM
 
 - `OPTIMIZE TABLE <table_name> PURGE`
 
-    Purges the historical data of table. Only the latest snapshot (including the segments and blocks referenced by this snapshot) will be kept.
+    Purges the historical data of table. Only the latest snapshot (including the segments, blocks and table statistic file referenced by this snapshot) will be kept.
 
 - `OPTIMIZE TABLE <table_name> COMPACT [LIMIT <segment_count>]`
 

src/query/service/tests/it/storages/fuse/operations/optimize.rs

@@ -63,14 +63,14 @@ async fn test_fuse_snapshot_optimize_statistic_purge() -> Result<()> {
         let ctx = fixture.ctx();
         execute_command(ctx, &qry).await?;
 
-        check_data_dir(&fixture, case_name, 3, 1 + i, 2, 2, 2, Some(())).await?;
+        check_data_dir(&fixture, case_name, 3, 1 + i, 2, 2, 2, Some(()), None).await?;
     }
 
     // After compact, all the count will become 1
     let qry = format!("optimize table {}.{} all", db, tbl);
     execute_command(fixture.ctx().clone(), &qry).await?;
 
-    check_data_dir(&fixture, case_name, 1, 1, 1, 1, 1, Some(())).await?;
+    check_data_dir(&fixture, case_name, 1, 1, 1, 1, 1, Some(()), Some(())).await?;
 
     Ok(())
 }
@@ -111,6 +111,7 @@ async fn do_purge_test(
         block_count,
         index_count,
         Some(()),
+        None,
     )
     .await?;
     history_should_have_item(&fixture, case_name, snapshot_count).await?;
@@ -130,6 +131,7 @@ async fn do_purge_test(
             block_count,
             index_count,
             Some(()),
+            None,
         )
         .await?;
 

src/query/service/tests/it/storages/fuse/operations/purge_drop.rs

@@ -59,6 +59,7 @@ async fn test_fuse_snapshot_truncate_in_drop_all_stmt() -> Result<()> {
         0, // 0 blocks
         0, // 0 index
         None,
+        None,
     )
     .await?;
     Ok(())

src/query/service/tests/it/storages/fuse/operations/purge_truncate.rs

@@ -43,6 +43,7 @@ async fn test_fuse_truncate_purge_stmt() -> Result<()> {
         2,
         expected_index_count,
         Some(()),
+        None,
     )
     .await?;
 
@@ -68,6 +69,7 @@ async fn test_fuse_truncate_purge_stmt() -> Result<()> {
         0,
         0,
         Some(()),
+        None,
     )
     .await?;
     Ok(())

src/query/service/tests/it/storages/fuse/table_test_fixture.rs

@@ -465,6 +465,7 @@ pub async fn check_data_dir(
     block_count: u32,
     index_count: u32,
     check_last_snapshot: Option<()>,
+    check_table_statistic_file: Option<()>,
 ) -> Result<()> {
     let data_path = match fixture.ctx().get_config().storage.params {
         StorageParams::Fs(v) => v.root,
@@ -477,6 +478,7 @@ pub async fn check_data_dir(
     let mut b_count = 0;
     let mut i_count = 0;
     let mut last_snapshot_loc = "".to_string();
+    let mut table_statistic_files = vec![];
     let prefix_snapshot = FUSE_TBL_SNAPSHOT_PREFIX;
     let prefix_snapshot_statistics = FUSE_TBL_SNAPSHOT_STATISTICS_PREFIX;
     let prefix_segment = FUSE_TBL_SEGMENT_PREFIX;
@@ -500,6 +502,7 @@ pub async fn check_data_dir(
                 i_count += 1;
             } else if path.starts_with(prefix_snapshot_statistics) {
                 ts_count += 1;
+                table_statistic_files.push(entry_path.to_string());
             } else if path.starts_with(prefix_last_snapshot_hint) && check_last_snapshot.is_some() {
                 let content = fixture
                     .ctx
@@ -552,6 +555,26 @@ pub async fn check_data_dir(
         );
     }
 
+    if check_table_statistic_file.is_some() {
+        let table = fixture.latest_default_table().await?;
+        let fuse_table = FuseTable::try_from_table(table.as_ref())?;
+        let snapshot_opt = fuse_table.read_table_snapshot().await?;
+        assert!(snapshot_opt.is_some());
+        let snapshot = snapshot_opt.unwrap();
+        let ts_location_opt = snapshot.table_statistics_location.clone();
+        assert!(ts_location_opt.is_some());
+        let ts_location = ts_location_opt.unwrap();
+        println!(
+            "ts_location_opt: {:?}, table_statistic_files: {:?}",
+            ts_location, table_statistic_files
+        );
+        assert!(
+            table_statistic_files
+                .iter()
+                .any(|e| e.contains(&ts_location))
+        );
+    }
+
     Ok(())
 }