Skip to content

Commit d9fbc5c

Browse files
ZENOTMEZENOTME
andauthored
refine: remove data_file_content in FileScanTask (#1485)
## Which issue does this PR close? I think we don't need data_file_content in FileScanTask seems it's always be `Data` and the delete file will be stored in `deletes`. ## What changes are included in this PR? ## Are these changes tested? Co-authored-by: ZENOTME <st810918843@gmail.com>
1 parent a3bf829 commit d9fbc5c

File tree

5 files changed

+1
-12
lines changed

5 files changed

+1
-12
lines changed

crates/iceberg/src/arrow/delete_filter.rs

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -334,7 +334,6 @@ pub(crate) mod tests {
334334
length: 0,
335335
record_count: None,
336336
data_file_path: format!("{}/1.parquet", table_location.to_str().unwrap()),
337-
data_file_content: DataContentType::Data,
338337
data_file_format: DataFileFormat::Parquet,
339338
schema: data_file_schema.clone(),
340339
project_field_ids: vec![],
@@ -346,7 +345,6 @@ pub(crate) mod tests {
346345
length: 0,
347346
record_count: None,
348347
data_file_path: format!("{}/2.parquet", table_location.to_str().unwrap()),
349-
data_file_content: DataContentType::Data,
350348
data_file_format: DataFileFormat::Parquet,
351349
schema: data_file_schema.clone(),
352350
project_field_ids: vec![],

crates/iceberg/src/arrow/reader.rs

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1451,9 +1451,7 @@ mod tests {
14511451
use crate::expr::{Bind, Predicate, Reference};
14521452
use crate::io::FileIO;
14531453
use crate::scan::{FileScanTask, FileScanTaskStream};
1454-
use crate::spec::{
1455-
DataContentType, DataFileFormat, Datum, NestedField, PrimitiveType, Schema, SchemaRef, Type,
1456-
};
1454+
use crate::spec::{DataFileFormat, Datum, NestedField, PrimitiveType, Schema, SchemaRef, Type};
14571455

14581456
fn table_schema_simple() -> SchemaRef {
14591457
Arc::new(
@@ -1742,7 +1740,6 @@ message schema {
17421740
length: 0,
17431741
record_count: None,
17441742
data_file_path: format!("{}/1.parquet", table_location),
1745-
data_file_content: DataContentType::Data,
17461743
data_file_format: DataFileFormat::Parquet,
17471744
schema: schema.clone(),
17481745
project_field_ids: vec![1],

crates/iceberg/src/scan/context.rs

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -119,7 +119,6 @@ impl ManifestEntryContext {
119119
record_count: Some(self.manifest_entry.record_count()),
120120

121121
data_file_path: self.manifest_entry.file_path().to_string(),
122-
data_file_content: self.manifest_entry.content_type(),
123122
data_file_format: self.manifest_entry.file_format(),
124123

125124
schema: self.snapshot_schema,

crates/iceberg/src/scan/mod.rs

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1777,7 +1777,6 @@ pub mod tests {
17771777
);
17781778
let task = FileScanTask {
17791779
data_file_path: "data_file_path".to_string(),
1780-
data_file_content: DataContentType::Data,
17811780
start: 0,
17821781
length: 100,
17831782
project_field_ids: vec![1, 2, 3],
@@ -1792,7 +1791,6 @@ pub mod tests {
17921791
// with predicate
17931792
let task = FileScanTask {
17941793
data_file_path: "data_file_path".to_string(),
1795-
data_file_content: DataContentType::Data,
17961794
start: 0,
17971795
length: 100,
17981796
project_field_ids: vec![1, 2, 3],

crates/iceberg/src/scan/task.rs

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -41,9 +41,6 @@ pub struct FileScanTask {
4141
/// The data file path corresponding to the task.
4242
pub data_file_path: String,
4343

44-
/// The content type of the file to scan.
45-
pub data_file_content: DataContentType,
46-
4744
/// The format of the file to scan.
4845
pub data_file_format: DataFileFormat,
4946

0 commit comments

Comments
 (0)