first set of changes

Author: rambleraptor

crates/iceberg/src/avro/schema.rs

@@ -116,23 +116,25 @@ impl SchemaVisitor for SchemaToAvroSchema {
             field_schema = avro_optional(field_schema)?;
         }
 
+        let default = if let Some(default) = &field.initial_default {
+            Some(literal_to_json(default)?)
+        } else if !field.required {
+            Some(Value::Null)
+        } else {
+            None
+        };
+
         let mut avro_record_field = AvroRecordField {
             name: field.name.clone(),
             schema: field_schema,
             order: RecordFieldOrder::Ignore,
             position: 0,
             doc: field.doc.clone(),
             aliases: None,
-            default: None,
+            default: default,
             custom_attributes: Default::default(),
         };
 
-        if let Some(default) = &field.initial_default {
-            avro_record_field.default = Some(literal_to_json(default)?);
-        } else if !field.required {
-            avro_record_field.default = Some(Value::Null);
-        }
-
         avro_record_field.custom_attributes.insert(
             FILED_ID_PROP.to_string(),
             Value::Number(Number::from(field.id)),

crates/iceberg/src/spec/manifest/_serde.rs

@@ -330,9 +330,8 @@ mod tests {
         assert_eq!(ret, expected_ret, "Negative i64 entry should be ignored!");
     }
 
-    #[tokio::test]
-    async fn test_data_file_serialize_deserialize() {
-        let schema = Arc::new(
+    fn schema() -> Arc<Schema> {
+        Arc::new(
             Schema::builder()
                 .with_fields(vec![
                     Arc::new(NestedField::optional(
@@ -353,8 +352,11 @@ mod tests {
                 ])
                 .build()
                 .unwrap(),
-        );
-        let data_files = vec![DataFile {
+        )
+    }
+
+    fn data_files() -> Vec<DataFile> {
+        vec![DataFile {
             content: DataContentType::Data,
             file_path: "s3://testbucket/iceberg_data/iceberg_ctl/iceberg_db/iceberg_tbl/data/00000-7-45268d71-54eb-476c-b42c-942d880c04a1-00001.parquet".to_string(),
             file_format: DataFileFormat::Parquet,
@@ -376,7 +378,13 @@ mod tests {
             referenced_data_file: None,
             content_offset: None,
             content_size_in_bytes: None,
-        }];
+        }]
+    }
+
+    #[tokio::test]
+    async fn test_data_file_serialize_deserialize() {
+        let schema = schema();
+        let data_files = data_files();
 
         let mut buffer = Vec::new();
         let _ = write_data_files_to_avro(
@@ -401,51 +409,8 @@ mod tests {
 
     #[tokio::test]
     async fn test_data_file_serialize_deserialize_v1_data_on_v2_reader() {
-        let schema = Arc::new(
-            Schema::builder()
-                .with_fields(vec![
-                    Arc::new(NestedField::optional(
-                        1,
-                        "v1",
-                        Type::Primitive(PrimitiveType::Int),
-                    )),
-                    Arc::new(NestedField::optional(
-                        2,
-                        "v2",
-                        Type::Primitive(PrimitiveType::String),
-                    )),
-                    Arc::new(NestedField::optional(
-                        3,
-                        "v3",
-                        Type::Primitive(PrimitiveType::String),
-                    )),
-                ])
-                .build()
-                .unwrap(),
-        );
-        let data_files = vec![DataFile {
-            content: DataContentType::Data,
-            file_path: "s3://testbucket/iceberg_data/iceberg_ctl/iceberg_db/iceberg_tbl/data/00000-7-45268d71-54eb-476c-b42c-942d880c04a1-00001.parquet".to_string(),
-            file_format: DataFileFormat::Parquet,
-            partition: Struct::empty(),
-            record_count: 1,
-            file_size_in_bytes: 875,
-            column_sizes: HashMap::from([(1,47),(2,48),(3,52)]),
-            value_counts: HashMap::from([(1,1),(2,1),(3,1)]),
-            null_value_counts: HashMap::from([(1,0),(2,0),(3,0)]),
-            nan_value_counts: HashMap::new(),
-            lower_bounds: HashMap::from([(1,Datum::int(1)),(2,Datum::string("a")),(3,Datum::string("AC/DC"))]),
-            upper_bounds: HashMap::from([(1,Datum::int(1)),(2,Datum::string("a")),(3,Datum::string("AC/DC"))]),
-            key_metadata: None,
-            split_offsets: vec![4],
-            equality_ids: vec![],
-            sort_order_id: Some(0),
-            partition_spec_id: 0,
-            first_row_id: None,
-            referenced_data_file: None,
-            content_offset: None,
-            content_size_in_bytes: None,
-        }];
+        let schema = schema();
+        let data_files = data_files();
 
         let mut buffer = Vec::new();
         let _ = write_data_files_to_avro(

crates/iceberg/src/spec/manifest/entry.rs

@@ -234,6 +234,7 @@ static CONTENT: Lazy<NestedFieldRef> = {
     Lazy::new(|| {
         Arc::new(
             NestedField::required(134, "content", Type::Primitive(PrimitiveType::Int))
+                // 0 refers to DataContentType::DATA
                 .with_initial_default(Literal::Primitive(PrimitiveLiteral::Int(0))),
         )
     })