Skip to content

Commit 2f5e9a6

Browse files
SkyFan2002SkyFan2002
committed
fix nullable
1 parent 621da2d commit 2f5e9a6

File tree

1 file changed

+64
-13
lines changed

1 file changed

+64
-13
lines changed

‎src/query/storages/common/blocks/src/parquet_rs.rs

Lines changed: 64 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -161,10 +161,14 @@ fn can_apply_delta_binary_pack(
161161
if num_rows == 0 {
162162
return Ok(false);
163163
}
164-
let col = col.value.convert_to_full_column(&col.data_type, num_rows);
164+
let col = col
165+
.value
166+
.convert_to_full_column(&col.data_type, num_rows)
167+
.remove_nullable();
165168
match col.data_type().remove_nullable() {
166169
DataType::Number(NumberDataType::UInt8) => {
167170
let mut max_delta = 0;
171+
let mut min_delta = u8::MAX;
168172
let col = col.as_number().unwrap().as_u_int8().unwrap();
169173
let mut col_iter = col.iter();
170174
let mut prev = *col_iter.next().unwrap();
@@ -173,12 +177,16 @@ fn can_apply_delta_binary_pack(
173177
if delta > max_delta {
174178
max_delta = delta;
175179
}
180+
if delta < min_delta {
181+
min_delta = delta;
182+
}
176183
prev = v;
177184
}
178-
Ok(max_delta as i64 <= MAX_DELTA)
185+
Ok((max_delta - min_delta) as i64 <= MAX_DELTA)
179186
}
180187
DataType::Number(NumberDataType::UInt16) => {
181188
let mut max_delta = 0;
189+
let mut min_delta = u16::MAX;
182190
let col = col.as_number().unwrap().as_u_int16().unwrap();
183191
let mut col_iter = col.iter();
184192
let mut prev = *col_iter.next().unwrap();
@@ -187,12 +195,16 @@ fn can_apply_delta_binary_pack(
187195
if delta > max_delta {
188196
max_delta = delta;
189197
}
198+
if delta < min_delta {
199+
min_delta = delta;
200+
}
190201
prev = v;
191202
}
192-
Ok(max_delta as i64 <= MAX_DELTA)
203+
Ok((max_delta - min_delta) as i64 <= MAX_DELTA)
193204
}
194205
DataType::Number(NumberDataType::UInt32) => {
195206
let mut max_delta = 0;
207+
let mut min_delta = u32::MAX;
196208
let col = col.as_number().unwrap().as_u_int32().unwrap();
197209
let mut col_iter = col.iter();
198210
let mut prev = *col_iter.next().unwrap();
@@ -201,12 +213,16 @@ fn can_apply_delta_binary_pack(
201213
if delta > max_delta {
202214
max_delta = delta;
203215
}
216+
if delta < min_delta {
217+
min_delta = delta;
218+
}
204219
prev = v;
205220
}
206-
Ok(max_delta as i64 <= MAX_DELTA)
221+
Ok((max_delta - min_delta) as i64 <= MAX_DELTA)
207222
}
208223
DataType::Number(NumberDataType::UInt64) => {
209224
let mut max_delta = 0;
225+
let mut min_delta = u64::MAX;
210226
let col = col.as_number().unwrap().as_u_int64().unwrap();
211227
let mut col_iter = col.iter();
212228
let mut prev = *col_iter.next().unwrap();
@@ -215,12 +231,16 @@ fn can_apply_delta_binary_pack(
215231
if delta > max_delta {
216232
max_delta = delta;
217233
}
234+
if delta < min_delta {
235+
min_delta = delta;
236+
}
218237
prev = v;
219238
}
220-
Ok(max_delta as i64 <= MAX_DELTA)
239+
Ok((max_delta - min_delta) <= MAX_DELTA as u64)
221240
}
222241
DataType::Number(NumberDataType::Int8) => {
223242
let mut max_delta = 0;
243+
let mut min_delta = i8::MAX;
224244
let col = col.as_number().unwrap().as_int8().unwrap();
225245
let mut col_iter = col.iter();
226246
let mut prev = *col_iter.next().unwrap();
@@ -229,12 +249,16 @@ fn can_apply_delta_binary_pack(
229249
if delta > max_delta {
230250
max_delta = delta;
231251
}
252+
if delta < min_delta {
253+
min_delta = delta;
254+
}
232255
prev = v;
233256
}
234-
Ok(max_delta as i64 <= MAX_DELTA)
257+
Ok((max_delta - min_delta) as i64 <= MAX_DELTA)
235258
}
236259
DataType::Number(NumberDataType::Int16) => {
237260
let mut max_delta = 0;
261+
let mut min_delta = i16::MAX;
238262
let col = col.as_number().unwrap().as_int16().unwrap();
239263
let mut col_iter = col.iter();
240264
let mut prev = *col_iter.next().unwrap();
@@ -243,12 +267,16 @@ fn can_apply_delta_binary_pack(
243267
if delta > max_delta {
244268
max_delta = delta;
245269
}
270+
if delta < min_delta {
271+
min_delta = delta;
272+
}
246273
prev = v;
247274
}
248-
Ok(max_delta as i64 <= MAX_DELTA)
275+
Ok((max_delta - min_delta) as i64 <= MAX_DELTA)
249276
}
250277
DataType::Number(NumberDataType::Int32) => {
251278
let mut max_delta = 0;
279+
let mut min_delta = i32::MAX;
252280
let col = col.as_number().unwrap().as_int32().unwrap();
253281
let mut col_iter = col.iter();
254282
let mut prev = *col_iter.next().unwrap();
@@ -257,12 +285,16 @@ fn can_apply_delta_binary_pack(
257285
if delta > max_delta {
258286
max_delta = delta;
259287
}
288+
if delta < min_delta {
289+
min_delta = delta;
290+
}
260291
prev = v;
261292
}
262-
Ok(max_delta as i64 <= MAX_DELTA)
293+
Ok((max_delta - min_delta) as i64 <= MAX_DELTA)
263294
}
264295
DataType::Number(NumberDataType::Int64) => {
265296
let mut max_delta = 0;
297+
let mut min_delta = i64::MAX;
266298
let col = col.as_number().unwrap().as_int64().unwrap();
267299
let mut col_iter = col.iter();
268300
let mut prev = *col_iter.next().unwrap();
@@ -271,12 +303,16 @@ fn can_apply_delta_binary_pack(
271303
if delta > max_delta {
272304
max_delta = delta;
273305
}
306+
if delta < min_delta {
307+
min_delta = delta;
308+
}
274309
prev = v;
275310
}
276-
Ok(max_delta <= MAX_DELTA)
311+
Ok((max_delta - min_delta) <= MAX_DELTA)
277312
}
278313
DataType::Decimal(DecimalDataType::Decimal128(_)) => {
279314
let mut max_delta = 0;
315+
let mut min_delta = i128::MAX;
280316
let (col, _) = col.as_decimal().unwrap().as_decimal128().unwrap();
281317
let mut col_iter = col.iter();
282318
let mut prev = *col_iter.next().unwrap();
@@ -285,12 +321,16 @@ fn can_apply_delta_binary_pack(
285321
if delta > max_delta {
286322
max_delta = delta;
287323
}
324+
if delta < min_delta {
325+
min_delta = delta;
326+
}
288327
prev = v;
289328
}
290-
Ok(max_delta <= MAX_DELTA as i128)
329+
Ok((max_delta - min_delta) <= MAX_DELTA as i128)
291330
}
292331
DataType::Decimal(DecimalDataType::Decimal256(_)) => {
293332
let mut max_delta: I256 = I256::ZERO;
333+
let mut min_delta = I256::MAX;
294334
let (col, _) = col.as_decimal().unwrap().as_decimal256().unwrap();
295335
let mut col_iter = col.iter();
296336
let mut prev = *col_iter.next().unwrap();
@@ -299,12 +339,16 @@ fn can_apply_delta_binary_pack(
299339
if delta > max_delta {
300340
max_delta = delta;
301341
}
342+
if delta < min_delta {
343+
min_delta = delta;
344+
}
302345
prev = v;
303346
}
304-
Ok(max_delta <= I256::from(MAX_DELTA))
347+
Ok(max_delta - min_delta <= I256::from(MAX_DELTA))
305348
}
306349
DataType::Timestamp => {
307350
let mut max_delta = 0;
351+
let mut min_delta = i64::MAX;
308352
let col = col.as_timestamp().unwrap();
309353
let mut col_iter = col.iter();
310354
let mut prev = *col_iter.next().unwrap();
@@ -313,12 +357,16 @@ fn can_apply_delta_binary_pack(
313357
if delta > max_delta {
314358
max_delta = delta;
315359
}
360+
if delta < min_delta {
361+
min_delta = delta;
362+
}
316363
prev = v;
317364
}
318-
Ok(max_delta <= MAX_DELTA)
365+
Ok((max_delta - min_delta) <= MAX_DELTA)
319366
}
320367
DataType::Date => {
321368
let mut max_delta = 0;
369+
let mut min_delta = i32::MAX;
322370
let col = col.as_date().unwrap();
323371
let mut col_iter = col.iter();
324372
let mut prev = *col_iter.next().unwrap();
@@ -327,9 +375,12 @@ fn can_apply_delta_binary_pack(
327375
if delta > max_delta {
328376
max_delta = delta;
329377
}
378+
if delta < min_delta {
379+
min_delta = delta;
380+
}
330381
prev = v;
331382
}
332-
Ok(max_delta as i64 <= MAX_DELTA)
383+
Ok((max_delta - min_delta) as i64 <= MAX_DELTA)
333384
}
334385
_ => Err(ErrorCode::Internal(format!(
335386
"Unsupported data type for delta binary pack: {:?}",

0 commit comments

Comments
 (0)