Skip to content

Commit bd8d3c6

Browse files
committed
fix: slice array cache item according to batch size
1 parent b9e3169 commit bd8d3c6

File tree

1 file changed

+7
-3
lines changed
  • src/query/storages/fuse/src/io/read/block/parquet

1 file changed

+7
-3
lines changed

src/query/storages/fuse/src/io/read/block/parquet/mod.rs

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414

1515
use std::collections::HashMap;
1616

17+
use arrow_array::Array;
1718
use arrow_array::ArrayRef;
1819
use arrow_array::RecordBatch;
1920
use arrow_array::StructArray;
@@ -94,6 +95,7 @@ impl BlockReader {
9495

9596
let mut blocks = Vec::with_capacity(record_batches.len());
9697

98+
let mut offset = 0;
9799
for record_batch in record_batches {
98100
let num_rows_record_batch = record_batch.num_rows();
99101
let mut columns = Vec::with_capacity(self.projected_schema.fields.len());
@@ -118,7 +120,7 @@ impl BlockReader {
118120
// corresponding field, and a default value has been declared for
119121
// the corresponding field.
120122
//
121-
// Yes, it is too obscure, we need to polish it SOON.
123+
// It is too confusing, we need to polish it SOON.
122124

123125
let value = match column_chunks.get(&field.column_id) {
124126
Some(DataItem::RawData(data)) => {
@@ -140,19 +142,21 @@ impl BlockReader {
140142
Value::from_arrow_rs(arrow_array, &data_type)?
141143
}
142144
Some(DataItem::ColumnArray(cached)) => {
143-
// TODO this is NOT correct!
144145
if column_node.is_nested {
145146
// a defensive check, should never happen
146147
return Err(ErrorCode::StorageOther(
147148
"unexpected nested field: nested leaf field hits cached",
148149
));
149150
}
150-
Value::from_arrow_rs(cached.0.clone(), &data_type)?
151+
let array = cached.0.slice(offset, record_batch.num_rows());
152+
Value::from_arrow_rs(array, &data_type)?
151153
}
152154
None => Value::Scalar(self.default_vals[i].clone()),
153155
};
154156
columns.push(BlockEntry::new(data_type, value));
155157
}
158+
159+
offset += record_batch.num_rows();
156160
blocks.push(DataBlock::new(columns, num_rows_record_batch));
157161
}
158162

0 commit comments

Comments
 (0)