refactor: refine column statistic map serialization (#16728)

While serializing ColumnStatistics map, excludes columns which is of unsupported type

Author: dantengsky

Cargo.lock

@@ -15,16 +15,5 @@
 use databend_common_expression::types::DataType;
 
 pub trait Index {
-    fn supported_type(data_type: &DataType) -> bool {
-        // we support nullable column but Nulls are not added into the bloom filter.
-        let inner_type = data_type.remove_nullable();
-        matches!(
-            inner_type,
-            DataType::Number(_)
-                | DataType::Date
-                | DataType::Timestamp
-                | DataType::String
-                | DataType::Decimal(_)
-        )
-    }
+    fn supported_type(data_type: &DataType) -> bool;
 }

src/query/storages/common/index/src/index.rs

@@ -247,4 +247,8 @@ pub fn statistics_to_domain(mut stats: Vec<&ColumnStatistics>, data_type: &DataT
     }
 }
 
-impl Index for RangeIndex {}
+impl Index for RangeIndex {
+    fn supported_type(data_type: &DataType) -> bool {
+        databend_storages_common_table_meta::meta::supported_stat_type(data_type)
+    }
+}

src/query/storages/common/index/src/range_index.rs

@@ -20,6 +20,7 @@ databend-common-expression = { workspace = true }
 databend-common-io = { workspace = true }
 databend-common-storage = { path = "../../../../common/storage" }
 enum-as-inner = "0.5"
+log = { workspace = true }
 parquet = { workspace = true }
 rmp-serde = "1.1.1"
 serde = { workspace = true }

src/query/storages/common/table_meta/Cargo.toml

@@ -15,6 +15,7 @@
 use std::collections::HashMap;
 
 use databend_common_base::base::uuid::Uuid;
+use databend_common_expression::types::DataType;
 use databend_common_expression::ColumnId;
 
 use crate::meta::ColumnStatistics;
@@ -36,3 +37,15 @@ pub struct BlockSlotDescription {
     // otherwise, the block should be taken care of by other executors
     pub slot: u32,
 }
+
+pub fn supported_stat_type(data_type: &DataType) -> bool {
+    let inner_type = data_type.remove_nullable();
+    matches!(
+        inner_type,
+        DataType::Number(_)
+            | DataType::Date
+            | DataType::Timestamp
+            | DataType::String
+            | DataType::Decimal(_)
+    )
+}

src/query/storages/common/table_meta/src/meta/statistics.rs

@@ -14,7 +14,6 @@
 
 use std::collections::HashMap;
 use std::fmt;
-use std::hash::Hash;
 use std::marker::PhantomData;
 
 use databend_common_expression::converts::datavalues::from_scalar;
@@ -24,8 +23,10 @@ use databend_common_expression::ColumnId;
 use databend_common_expression::Scalar;
 use databend_common_expression::TableDataType;
 use databend_common_expression::TableField;
+use log::info;
 use serde::de::Error;
 
+use crate::meta::supported_stat_type;
 use crate::meta::v0;
 
 #[derive(serde::Serialize, serde::Deserialize, Debug, Clone, PartialEq, Eq)]
@@ -367,25 +368,20 @@ pub fn deserialize_col_stats<'de, D>(
 where D: serde::Deserializer<'de> {
     deserializer.deserialize_map(ColStatsVisitor::new())
 }
-
-struct ColStatsVisitor<K, V> {
-    marker: PhantomData<fn() -> HashMap<K, V>>,
+struct ColStatsVisitor {
+    marker: PhantomData<fn() -> HashMap<ColumnId, ColumnStatistics>>,
 }
 
-impl<K, V> ColStatsVisitor<K, V> {
+impl ColStatsVisitor {
     fn new() -> Self {
         ColStatsVisitor {
             marker: PhantomData,
         }
     }
 }
 
-impl<'de, K, V> serde::de::Visitor<'de> for ColStatsVisitor<K, V>
-where
-    K: serde::Deserialize<'de> + Hash + Eq,
-    V: serde::Deserialize<'de>,
-{
-    type Value = HashMap<K, V>;
+impl<'de> serde::de::Visitor<'de> for ColStatsVisitor {
+    type Value = HashMap<ColumnId, ColumnStatistics>;
 
     fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
         formatter.write_str("a map")
@@ -395,9 +391,17 @@ where
     where M: serde::de::MapAccess<'de> {
         let mut map = HashMap::with_capacity(access.size_hint().unwrap_or(0));
 
-        while let Some(key) = access.next_key()? {
-            if let Ok(value) = access.next_value() {
-                map.insert(key, value);
+        while let Some(key) = access.next_key::<ColumnId>()? {
+            if let Ok(value) = access.next_value::<ColumnStatistics>() {
+                let data_type = value.max.as_ref().infer_data_type();
+                if supported_stat_type(&data_type) {
+                    map.insert(key, value);
+                } else {
+                    info!(
+                        "column of id {} is excluded from column statistics, unsupported data type {}",
+                        key, data_type
+                    );
+                }
             }
         }