Skip to content

Commit 621da2d

Browse files
SkyFan2002SkyFan2002
committed
try enable delte binary packed encoding
1 parent 9bfff0c commit 621da2d

File tree

3 files changed

+203
-2
lines changed

3 files changed

+203
-2
lines changed

‎Cargo.lock

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎src/query/storages/common/blocks/Cargo.toml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,8 +14,8 @@ test = false
1414
[dependencies]
1515
databend-common-exception = { path = "../../../../common/exception" }
1616
databend-common-expression = { path = "../../../expression" }
17-
parquet_rs = { workspace = true }
18-
1917
databend-storages-common-table-meta = { path = "../table_meta" }
18+
ethnum = { workspace = true }
19+
parquet_rs = { workspace = true }
2020

2121
[build-dependencies]

‎src/query/storages/common/blocks/src/parquet_rs.rs

Lines changed: 200 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,13 +14,18 @@
1414

1515
use std::sync::Arc;
1616

17+
use databend_common_exception::ErrorCode;
1718
use databend_common_exception::Result;
1819
use databend_common_expression::converts::arrow::table_schema_to_arrow_schema_ignore_inside_nullable;
20+
use databend_common_expression::types::DataType;
21+
use databend_common_expression::types::DecimalDataType;
22+
use databend_common_expression::types::NumberDataType;
1923
use databend_common_expression::BlockEntry;
2024
use databend_common_expression::DataBlock;
2125
use databend_common_expression::TableSchema;
2226
use databend_storages_common_table_meta::meta::StatisticsOfColumns;
2327
use databend_storages_common_table_meta::table::TableCompression;
28+
use ethnum::I256;
2429
use parquet_rs::arrow::arrow_to_parquet_schema;
2530
use parquet_rs::arrow::ArrowWriter;
2631
use parquet_rs::basic::Encoding;
@@ -107,6 +112,9 @@ fn choose_compression_scheme(
107112
if can_apply_dict_encoding(physical_type, distinct_of_values, num_rows, col)? {
108113
let col_path = ColumnPath::new(vec![table_field.name().clone()]);
109114
props = props.set_column_dictionary_enabled(col_path, true);
115+
} else if can_apply_delta_binary_pack(physical_type, col, num_rows)? {
116+
let col_path = ColumnPath::new(vec![table_field.name().clone()]);
117+
props = props.set_column_encoding(col_path, Encoding::DELTA_BINARY_PACKED);
110118
}
111119
}
112120
Type::GroupType {
@@ -137,3 +145,195 @@ fn can_apply_dict_encoding(
137145
let avg_bytes_per_value = total_bytes as f64 / num_rows as f64;
138146
Ok(is_low_cardinality && avg_bytes_per_value < AVG_BYTES_PER_VALUE)
139147
}
148+
149+
fn can_apply_delta_binary_pack(
150+
physical_type: &PhysicalType,
151+
col: &BlockEntry,
152+
num_rows: usize,
153+
) -> Result<bool> {
154+
const MAX_DELTA: i64 = 1 << 3;
155+
if !matches!(
156+
physical_type,
157+
PhysicalType::INT32 | PhysicalType::INT64 | PhysicalType::INT96
158+
) {
159+
return Ok(false);
160+
}
161+
if num_rows == 0 {
162+
return Ok(false);
163+
}
164+
let col = col.value.convert_to_full_column(&col.data_type, num_rows);
165+
match col.data_type().remove_nullable() {
166+
DataType::Number(NumberDataType::UInt8) => {
167+
let mut max_delta = 0;
168+
let col = col.as_number().unwrap().as_u_int8().unwrap();
169+
let mut col_iter = col.iter();
170+
let mut prev = *col_iter.next().unwrap();
171+
for &v in col_iter {
172+
let delta = if v > prev { v - prev } else { prev - v };
173+
if delta > max_delta {
174+
max_delta = delta;
175+
}
176+
prev = v;
177+
}
178+
Ok(max_delta as i64 <= MAX_DELTA)
179+
}
180+
DataType::Number(NumberDataType::UInt16) => {
181+
let mut max_delta = 0;
182+
let col = col.as_number().unwrap().as_u_int16().unwrap();
183+
let mut col_iter = col.iter();
184+
let mut prev = *col_iter.next().unwrap();
185+
for &v in col_iter {
186+
let delta = if v > prev { v - prev } else { prev - v };
187+
if delta > max_delta {
188+
max_delta = delta;
189+
}
190+
prev = v;
191+
}
192+
Ok(max_delta as i64 <= MAX_DELTA)
193+
}
194+
DataType::Number(NumberDataType::UInt32) => {
195+
let mut max_delta = 0;
196+
let col = col.as_number().unwrap().as_u_int32().unwrap();
197+
let mut col_iter = col.iter();
198+
let mut prev = *col_iter.next().unwrap();
199+
for &v in col_iter {
200+
let delta = if v > prev { v - prev } else { prev - v };
201+
if delta > max_delta {
202+
max_delta = delta;
203+
}
204+
prev = v;
205+
}
206+
Ok(max_delta as i64 <= MAX_DELTA)
207+
}
208+
DataType::Number(NumberDataType::UInt64) => {
209+
let mut max_delta = 0;
210+
let col = col.as_number().unwrap().as_u_int64().unwrap();
211+
let mut col_iter = col.iter();
212+
let mut prev = *col_iter.next().unwrap();
213+
for &v in col_iter {
214+
let delta = if v > prev { v - prev } else { prev - v };
215+
if delta > max_delta {
216+
max_delta = delta;
217+
}
218+
prev = v;
219+
}
220+
Ok(max_delta as i64 <= MAX_DELTA)
221+
}
222+
DataType::Number(NumberDataType::Int8) => {
223+
let mut max_delta = 0;
224+
let col = col.as_number().unwrap().as_int8().unwrap();
225+
let mut col_iter = col.iter();
226+
let mut prev = *col_iter.next().unwrap();
227+
for &v in col_iter {
228+
let delta = if v > prev { v - prev } else { prev - v };
229+
if delta > max_delta {
230+
max_delta = delta;
231+
}
232+
prev = v;
233+
}
234+
Ok(max_delta as i64 <= MAX_DELTA)
235+
}
236+
DataType::Number(NumberDataType::Int16) => {
237+
let mut max_delta = 0;
238+
let col = col.as_number().unwrap().as_int16().unwrap();
239+
let mut col_iter = col.iter();
240+
let mut prev = *col_iter.next().unwrap();
241+
for &v in col_iter {
242+
let delta = if v > prev { v - prev } else { prev - v };
243+
if delta > max_delta {
244+
max_delta = delta;
245+
}
246+
prev = v;
247+
}
248+
Ok(max_delta as i64 <= MAX_DELTA)
249+
}
250+
DataType::Number(NumberDataType::Int32) => {
251+
let mut max_delta = 0;
252+
let col = col.as_number().unwrap().as_int32().unwrap();
253+
let mut col_iter = col.iter();
254+
let mut prev = *col_iter.next().unwrap();
255+
for &v in col_iter {
256+
let delta = if v > prev { v - prev } else { prev - v };
257+
if delta > max_delta {
258+
max_delta = delta;
259+
}
260+
prev = v;
261+
}
262+
Ok(max_delta as i64 <= MAX_DELTA)
263+
}
264+
DataType::Number(NumberDataType::Int64) => {
265+
let mut max_delta = 0;
266+
let col = col.as_number().unwrap().as_int64().unwrap();
267+
let mut col_iter = col.iter();
268+
let mut prev = *col_iter.next().unwrap();
269+
for &v in col_iter {
270+
let delta = if v > prev { v - prev } else { prev - v };
271+
if delta > max_delta {
272+
max_delta = delta;
273+
}
274+
prev = v;
275+
}
276+
Ok(max_delta <= MAX_DELTA)
277+
}
278+
DataType::Decimal(DecimalDataType::Decimal128(_)) => {
279+
let mut max_delta = 0;
280+
let (col, _) = col.as_decimal().unwrap().as_decimal128().unwrap();
281+
let mut col_iter = col.iter();
282+
let mut prev = *col_iter.next().unwrap();
283+
for &v in col_iter {
284+
let delta = if v > prev { v - prev } else { prev - v };
285+
if delta > max_delta {
286+
max_delta = delta;
287+
}
288+
prev = v;
289+
}
290+
Ok(max_delta <= MAX_DELTA as i128)
291+
}
292+
DataType::Decimal(DecimalDataType::Decimal256(_)) => {
293+
let mut max_delta: I256 = I256::ZERO;
294+
let (col, _) = col.as_decimal().unwrap().as_decimal256().unwrap();
295+
let mut col_iter = col.iter();
296+
let mut prev = *col_iter.next().unwrap();
297+
for &v in col_iter {
298+
let delta = if v > prev { v - prev } else { prev - v };
299+
if delta > max_delta {
300+
max_delta = delta;
301+
}
302+
prev = v;
303+
}
304+
Ok(max_delta <= I256::from(MAX_DELTA))
305+
}
306+
DataType::Timestamp => {
307+
let mut max_delta = 0;
308+
let col = col.as_timestamp().unwrap();
309+
let mut col_iter = col.iter();
310+
let mut prev = *col_iter.next().unwrap();
311+
for &v in col_iter {
312+
let delta = if v > prev { v - prev } else { prev - v };
313+
if delta > max_delta {
314+
max_delta = delta;
315+
}
316+
prev = v;
317+
}
318+
Ok(max_delta <= MAX_DELTA)
319+
}
320+
DataType::Date => {
321+
let mut max_delta = 0;
322+
let col = col.as_date().unwrap();
323+
let mut col_iter = col.iter();
324+
let mut prev = *col_iter.next().unwrap();
325+
for &v in col_iter {
326+
let delta = if v > prev { v - prev } else { prev - v };
327+
if delta > max_delta {
328+
max_delta = delta;
329+
}
330+
prev = v;
331+
}
332+
Ok(max_delta as i64 <= MAX_DELTA)
333+
}
334+
_ => Err(ErrorCode::Internal(format!(
335+
"Unsupported data type for delta binary pack: {:?}",
336+
col.data_type()
337+
))),
338+
}
339+
}

0 commit comments

Comments
 (0)