From 561c7800a1a2e4f3ceec8ae7c6dc5061d286da38 Mon Sep 17 00:00:00 2001 From: David Sherrier Date: Sun, 11 May 2025 20:53:51 +0100 Subject: [PATCH 1/5] feat: Align object array to collection serialization protocol v2 #1229 --- .../fury/serializer/ArraySerializers.java | 204 ++++++++++++------ .../collection/FuryArrayAsListSerializer.java | 7 + .../fury/serializer/ArraySerializersTest.java | 1 + 3 files changed, 143 insertions(+), 69 deletions(-) diff --git a/java/fury-core/src/main/java/org/apache/fury/serializer/ArraySerializers.java b/java/fury-core/src/main/java/org/apache/fury/serializer/ArraySerializers.java index 4306cbabae..45afda4ae8 100644 --- a/java/fury-core/src/main/java/org/apache/fury/serializer/ArraySerializers.java +++ b/java/fury-core/src/main/java/org/apache/fury/serializer/ArraySerializers.java @@ -22,16 +22,22 @@ import java.lang.reflect.Array; import java.util.Arrays; import java.util.IdentityHashMap; +import java.util.List; import org.apache.fury.Fury; import org.apache.fury.config.CompatibleMode; import org.apache.fury.memory.MemoryBuffer; import org.apache.fury.memory.Platform; +import org.apache.fury.reflect.TypeRef; import org.apache.fury.resolver.ClassInfo; import org.apache.fury.resolver.ClassInfoHolder; import org.apache.fury.resolver.ClassResolver; import org.apache.fury.resolver.RefResolver; +import org.apache.fury.serializer.NonexistentClass.NonexistentSkip; +import org.apache.fury.serializer.NonexistentClassSerializers.NonexistentEnumClassSerializer; +import org.apache.fury.serializer.Serializers.CrossLanguageCompatibleSerializer; import org.apache.fury.serializer.collection.CollectionFlags; import org.apache.fury.serializer.collection.FuryArrayAsListSerializer; +import org.apache.fury.serializer.collection.FuryArrayAsListSerializer.ArrayAsList; import org.apache.fury.type.GenericType; import org.apache.fury.type.TypeUtils; import org.apache.fury.type.Types; @@ -45,6 +51,8 @@ public class ArraySerializers { public static final class ObjectArraySerializer extends Serializer { private final Class innerType; private final Serializer componentTypeSerializer; + private final FuryArrayAsListSerializer collectionSerializer; + private final GenericType collectionGenericType; private final ClassInfoHolder classInfoHolder; private final int[] stubDims; private final GenericType componentGenericType; @@ -69,12 +77,23 @@ public ObjectArraySerializer(Fury fury, Class cls) { if (fury.getClassResolver().isMonomorphic(componentType)) { if (fury.isCrossLanguage()) { this.componentTypeSerializer = null; + this.collectionSerializer = null; + this.collectionGenericType = null; } else { this.componentTypeSerializer = fury.getClassResolver().getSerializer(componentType); + if (dimension > 1) { + this.collectionSerializer = new FuryArrayAsListSerializer(fury); + this.collectionGenericType = buildCollectionGenericType(dimension); + } else { + this.collectionSerializer = null; + this.collectionGenericType = null; + } } } else { // TODO add ClassInfo cache for non-final component type. this.componentTypeSerializer = null; + this.collectionSerializer = null; + this.collectionGenericType = null; } this.stubDims = new int[dimension]; classInfoHolder = fury.getClassResolver().nilClassInfoHolder(); @@ -85,27 +104,35 @@ public void write(MemoryBuffer buffer, T[] arr) { int len = arr.length; RefResolver refResolver = fury.getRefResolver(); Serializer componentSerializer = this.componentTypeSerializer; - int header = componentSerializer != null ? 0b1 : 0b0; - buffer.writeVarUint32Small7(len << 1 | header); - if (componentSerializer != null) { - for (T t : arr) { - if (!refResolver.writeRefOrNull(buffer, t)) { - componentSerializer.write(buffer, t); - } - } + if (this.collectionSerializer != null) { + fury.getGenerics().pushGenericType(this.collectionGenericType); + ArrayAsList list = new ArrayAsList(0); + list.setArray(arr); + this.collectionSerializer.write(buffer, list); + fury.getGenerics().popGenericType(); } else { - Fury fury = this.fury; - ClassResolver classResolver = fury.getClassResolver(); - ClassInfo classInfo = null; - Class elemClass = null; - for (T t : arr) { - if (!refResolver.writeRefOrNull(buffer, t)) { - Class clz = t.getClass(); - if (clz != elemClass) { - elemClass = clz; - classInfo = classResolver.getClassInfo(clz); + int header = componentSerializer != null ? 0b1 : 0b0; + buffer.writeVarUint32Small7(len << 1 | header); + if (componentSerializer != null) { + for (T t : arr) { + if (!refResolver.writeRefOrNull(buffer, t)) { + componentSerializer.write(buffer, t); + } + } + } else { + Fury fury = this.fury; + ClassResolver classResolver = fury.getClassResolver(); + ClassInfo classInfo = null; + Class elemClass = null; + for (T t : arr) { + if (!refResolver.writeRefOrNull(buffer, t)) { + Class clz = t.getClass(); + if (clz != elemClass) { + elemClass = clz; + classInfo = classResolver.getClassInfo(clz); + } + fury.writeNonRef(buffer, t, classInfo); } - fury.writeNonRef(buffer, t, classInfo); } } } @@ -113,24 +140,33 @@ public void write(MemoryBuffer buffer, T[] arr) { @Override public T[] copy(T[] originArray) { - int length = originArray.length; - Object[] newArray = newArray(length); - if (needToCopyRef) { - fury.reference(originArray, newArray); - } + Object[] newArray; Serializer componentSerializer = this.componentTypeSerializer; - if (componentSerializer != null) { - if (componentSerializer.isImmutable()) { - System.arraycopy(originArray, 0, newArray, 0, length); + if (this.collectionSerializer != null) { + fury.getGenerics().pushGenericType(this.collectionGenericType); + ArrayAsList list = new ArrayAsList(originArray.length); + list.setArray(originArray); + newArray = this.collectionSerializer.copy(list).toArray(); + fury.getGenerics().popGenericType(); + } else { + int length = originArray.length; + newArray = newArray(length); + if (needToCopyRef) { + fury.reference(originArray, newArray); + } + if (componentSerializer != null) { + if (componentSerializer.isImmutable()) { + System.arraycopy(originArray, 0, newArray, 0, length); + } else { + for (int i = 0; i < length; i++) { + newArray[i] = componentSerializer.copy(originArray[i]); + } + } } else { for (int i = 0; i < length; i++) { - newArray[i] = componentSerializer.copy(originArray[i]); + newArray[i] = fury.copyObject(originArray[i]); } } - } else { - for (int i = 0; i < length; i++) { - newArray[i] = fury.copyObject(originArray[i]); - } } return (T[]) newArray; } @@ -147,39 +183,46 @@ public void xwrite(MemoryBuffer buffer, T[] arr) { @Override public T[] read(MemoryBuffer buffer) { - int numElements = buffer.readVarUint32Small7(); - boolean isFinal = (numElements & 0b1) != 0; - numElements >>>= 1; - Object[] value = newArray(numElements); - RefResolver refResolver = fury.getRefResolver(); - refResolver.reference(value); - if (isFinal) { - final Serializer componentTypeSerializer = this.componentTypeSerializer; - for (int i = 0; i < numElements; i++) { - Object elem; - int nextReadRefId = refResolver.tryPreserveRefId(buffer); - if (nextReadRefId >= Fury.NOT_NULL_VALUE_FLAG) { - elem = componentTypeSerializer.read(buffer); - refResolver.setReadObject(nextReadRefId, elem); - } else { - elem = refResolver.getReadObject(); - } - value[i] = elem; - } + Object[] value; + if (this.collectionSerializer != null) { + fury.getGenerics().pushGenericType(this.collectionGenericType); + value = this.collectionSerializer.read(buffer).toArray(); + fury.getGenerics().popGenericType(); } else { - Fury fury = this.fury; - ClassInfoHolder classInfoHolder = this.classInfoHolder; - for (int i = 0; i < numElements; i++) { - int nextReadRefId = refResolver.tryPreserveRefId(buffer); - Object o; - if (nextReadRefId >= Fury.NOT_NULL_VALUE_FLAG) { - // ref value or not-null value - o = fury.readNonRef(buffer, classInfoHolder); - refResolver.setReadObject(nextReadRefId, o); - } else { - o = refResolver.getReadObject(); + int numElements = buffer.readVarUint32Small7(); + boolean isFinal = (numElements & 0b1) != 0; + numElements >>>= 1; + value = newArray(numElements); + RefResolver refResolver = fury.getRefResolver(); + refResolver.reference(value); + if (isFinal) { + final Serializer componentTypeSerializer = this.componentTypeSerializer; + for (int i = 0; i < numElements; i++) { + Object elem; + int nextReadRefId = refResolver.tryPreserveRefId(buffer); + if (nextReadRefId >= Fury.NOT_NULL_VALUE_FLAG) { + elem = componentTypeSerializer.read(buffer); + refResolver.setReadObject(nextReadRefId, elem); + } else { + elem = refResolver.getReadObject(); + } + value[i] = elem; + } + } else { + Fury fury = this.fury; + ClassInfoHolder classInfoHolder = this.classInfoHolder; + for (int i = 0; i < numElements; i++) { + int nextReadRefId = refResolver.tryPreserveRefId(buffer); + Object o; + if (nextReadRefId >= Fury.NOT_NULL_VALUE_FLAG) { + // ref value or not-null value + o = fury.readNonRef(buffer, classInfoHolder); + refResolver.setReadObject(nextReadRefId, o); + } else { + o = refResolver.getReadObject(); + } + value[i] = o; } - value[i] = o; } } return (T[]) value; @@ -208,6 +251,30 @@ private Object[] newArray(int numElements) { } return value; } + + private GenericType buildCollectionGenericType(int dimensions) { + GenericType genericType; + switch (dimensions) { + case 1: + genericType = GenericType.build(new TypeRef>() {}); + break; + case 2: + genericType = GenericType.build(new TypeRef>>() {}); + break; + case 3: + genericType = GenericType.build(new TypeRef>>>() {}); + break; + case 4: + genericType = GenericType.build(new TypeRef>>>>() {}); + break; + case 5: + genericType = GenericType.build(new TypeRef>>>>>() {}); + break; + default: + throw new IllegalStateException("Unexpected value: " + dimensions); + } + return genericType; + } } public static final class PrimitiveArrayBufferObject implements BufferObject { @@ -249,7 +316,7 @@ public MemoryBuffer toBuffer() { // Implement all read/write methods in subclasses to avoid // virtual method call cost. public abstract static class PrimitiveArraySerializer - extends Serializers.CrossLanguageCompatibleSerializer { + extends CrossLanguageCompatibleSerializer { protected final int offset; protected final int elemSize; @@ -610,14 +677,14 @@ public double[] read(MemoryBuffer buffer) { public static final class StringArraySerializer extends Serializer { private final StringSerializer stringSerializer; private final FuryArrayAsListSerializer collectionSerializer; - private final FuryArrayAsListSerializer.ArrayAsList list; + private final ArrayAsList list; public StringArraySerializer(Fury fury) { super(fury, String[].class); stringSerializer = new StringSerializer(fury); collectionSerializer = new FuryArrayAsListSerializer(fury); collectionSerializer.setElementSerializer(stringSerializer); - list = new FuryArrayAsListSerializer.ArrayAsList(0); + list = new ArrayAsList(0); } @Override @@ -890,11 +957,10 @@ public NonexistentArrayClassSerializer(Fury fury, Class cls) { public NonexistentArrayClassSerializer(Fury fury, String className, Class cls) { super(fury, className, cls); if (TypeUtils.getArrayComponent(cls).isEnum()) { - componentSerializer = new NonexistentClassSerializers.NonexistentEnumClassSerializer(fury); + componentSerializer = new NonexistentEnumClassSerializer(fury); } else { if (fury.getConfig().getCompatibleMode() == CompatibleMode.COMPATIBLE) { - componentSerializer = - new CompatibleSerializer<>(fury, NonexistentClass.NonexistentSkip.class); + componentSerializer = new CompatibleSerializer<>(fury, NonexistentSkip.class); } else { componentSerializer = null; } diff --git a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/FuryArrayAsListSerializer.java b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/FuryArrayAsListSerializer.java index 09268c4e15..86f22f27f6 100644 --- a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/FuryArrayAsListSerializer.java +++ b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/FuryArrayAsListSerializer.java @@ -42,6 +42,13 @@ public Collection newCollection(MemoryBuffer buffer) { return new ArrayAsList(numElements); } + @Override + public Collection newCollection(Collection collection) { + int numElements = collection.size(); + setNumElements(numElements); + return new ArrayAsList(numElements); + } + /** * A List which wrap a Java array into a list, used for serialization only, do not use it in other * scenarios. diff --git a/java/fury-core/src/test/java/org/apache/fury/serializer/ArraySerializersTest.java b/java/fury-core/src/test/java/org/apache/fury/serializer/ArraySerializersTest.java index 6eb438b41e..00b0898d5a 100644 --- a/java/fury-core/src/test/java/org/apache/fury/serializer/ArraySerializersTest.java +++ b/java/fury-core/src/test/java/org/apache/fury/serializer/ArraySerializersTest.java @@ -76,6 +76,7 @@ public void testObjectArraySerialization(boolean referenceTracking, Language lan fury1, fury2, new Object[] {false, true, (byte) 1, (byte) 1, (float) 1.0, (float) 1.1}); serDeCheckTyped(fury1, fury2, new String[] {"str", "str"}); serDeCheckTyped(fury1, fury2, new Object[] {"str", 1}); + serDeCheckTyped(fury1, fury2, new String[][] {{"str", "str"}, {"abc", "def"}}); } @Test(dataProvider = "furyCopyConfig") From 56002e2bab49e14ce5725fe299b31928b467b574 Mon Sep 17 00:00:00 2001 From: David Sherrier Date: Mon, 12 May 2025 11:15:53 +0100 Subject: [PATCH 2/5] fix: build collection type dynamically with TypeUtils --- .../fury/serializer/ArraySerializers.java | 27 ++++--------------- 1 file changed, 5 insertions(+), 22 deletions(-) diff --git a/java/fury-core/src/main/java/org/apache/fury/serializer/ArraySerializers.java b/java/fury-core/src/main/java/org/apache/fury/serializer/ArraySerializers.java index 45afda4ae8..96f3e94b68 100644 --- a/java/fury-core/src/main/java/org/apache/fury/serializer/ArraySerializers.java +++ b/java/fury-core/src/main/java/org/apache/fury/serializer/ArraySerializers.java @@ -22,7 +22,6 @@ import java.lang.reflect.Array; import java.util.Arrays; import java.util.IdentityHashMap; -import java.util.List; import org.apache.fury.Fury; import org.apache.fury.config.CompatibleMode; import org.apache.fury.memory.MemoryBuffer; @@ -252,28 +251,12 @@ private Object[] newArray(int numElements) { return value; } - private GenericType buildCollectionGenericType(int dimensions) { - GenericType genericType; - switch (dimensions) { - case 1: - genericType = GenericType.build(new TypeRef>() {}); - break; - case 2: - genericType = GenericType.build(new TypeRef>>() {}); - break; - case 3: - genericType = GenericType.build(new TypeRef>>>() {}); - break; - case 4: - genericType = GenericType.build(new TypeRef>>>>() {}); - break; - case 5: - genericType = GenericType.build(new TypeRef>>>>>() {}); - break; - default: - throw new IllegalStateException("Unexpected value: " + dimensions); + private GenericType buildCollectionGenericType(int dims) { + TypeRef arrayType = TypeRef.of(this.innerType); + for (int i = 0; i < dims; i++) { + arrayType = TypeUtils.collectionOf(arrayType); } - return genericType; + return GenericType.build(arrayType); } } From 3cdf8ef6d2366a7d0da710a7423ddef0764ffc9e Mon Sep 17 00:00:00 2001 From: David Sherrier Date: Mon, 12 May 2025 12:35:00 +0100 Subject: [PATCH 3/5] fix: cache ArrayAsList and use collectionm serializer for 1D objects as well --- .../fury/serializer/ArraySerializers.java | 21 ++++++++++++------- .../fury/serializer/ArraySerializersTest.java | 2 +- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/java/fury-core/src/main/java/org/apache/fury/serializer/ArraySerializers.java b/java/fury-core/src/main/java/org/apache/fury/serializer/ArraySerializers.java index 96f3e94b68..75155f0191 100644 --- a/java/fury-core/src/main/java/org/apache/fury/serializer/ArraySerializers.java +++ b/java/fury-core/src/main/java/org/apache/fury/serializer/ArraySerializers.java @@ -51,6 +51,7 @@ public static final class ObjectArraySerializer extends Serializer { private final Class innerType; private final Serializer componentTypeSerializer; private final FuryArrayAsListSerializer collectionSerializer; + private ArrayAsList list; private final GenericType collectionGenericType; private final ClassInfoHolder classInfoHolder; private final int[] stubDims; @@ -80,13 +81,8 @@ public ObjectArraySerializer(Fury fury, Class cls) { this.collectionGenericType = null; } else { this.componentTypeSerializer = fury.getClassResolver().getSerializer(componentType); - if (dimension > 1) { - this.collectionSerializer = new FuryArrayAsListSerializer(fury); - this.collectionGenericType = buildCollectionGenericType(dimension); - } else { - this.collectionSerializer = null; - this.collectionGenericType = null; - } + this.collectionSerializer = new FuryArrayAsListSerializer(fury); + this.collectionGenericType = buildCollectionGenericType(dimension); } } else { // TODO add ClassInfo cache for non-final component type. @@ -95,6 +91,7 @@ public ObjectArraySerializer(Fury fury, Class cls) { this.collectionGenericType = null; } this.stubDims = new int[dimension]; + this.list = null; classInfoHolder = fury.getClassResolver().nilClassInfoHolder(); } @@ -105,10 +102,18 @@ public void write(MemoryBuffer buffer, T[] arr) { Serializer componentSerializer = this.componentTypeSerializer; if (this.collectionSerializer != null) { fury.getGenerics().pushGenericType(this.collectionGenericType); - ArrayAsList list = new ArrayAsList(0); + ArrayAsList list = this.list; + if (list == null) { + list = new ArrayAsList(0); + } else { + this.list = null; + } list.setArray(arr); + fury.incDepth(1); this.collectionSerializer.write(buffer, list); + fury.incDepth(1); fury.getGenerics().popGenericType(); + this.list = list; } else { int header = componentSerializer != null ? 0b1 : 0b0; buffer.writeVarUint32Small7(len << 1 | header); diff --git a/java/fury-core/src/test/java/org/apache/fury/serializer/ArraySerializersTest.java b/java/fury-core/src/test/java/org/apache/fury/serializer/ArraySerializersTest.java index 00b0898d5a..a4bd908f52 100644 --- a/java/fury-core/src/test/java/org/apache/fury/serializer/ArraySerializersTest.java +++ b/java/fury-core/src/test/java/org/apache/fury/serializer/ArraySerializersTest.java @@ -76,7 +76,6 @@ public void testObjectArraySerialization(boolean referenceTracking, Language lan fury1, fury2, new Object[] {false, true, (byte) 1, (byte) 1, (float) 1.0, (float) 1.1}); serDeCheckTyped(fury1, fury2, new String[] {"str", "str"}); serDeCheckTyped(fury1, fury2, new Object[] {"str", 1}); - serDeCheckTyped(fury1, fury2, new String[][] {{"str", "str"}, {"abc", "def"}}); } @Test(dataProvider = "furyCopyConfig") @@ -120,6 +119,7 @@ public void testMultiArraySerialization(boolean referenceTracking, Language lang {false, true, (byte) 1, (byte) 1, (float) 1.0, (float) 1.1} }); serDeCheckTyped(fury1, fury2, new Integer[][] {{1, 2}, {1, 2}}); + serDeCheckTyped(fury1, fury2, new String[][][] {{{"str", "str"}, {"str", "str"}}}); } @Test(dataProvider = "furyCopyConfig") From 7e2d793368239418a57adaccc2448f373e0ca519 Mon Sep 17 00:00:00 2001 From: David Sherrier Date: Mon, 12 May 2025 13:15:33 +0100 Subject: [PATCH 4/5] fix: handle depth during write serialization --- .../main/java/org/apache/fury/serializer/ArraySerializers.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/fury-core/src/main/java/org/apache/fury/serializer/ArraySerializers.java b/java/fury-core/src/main/java/org/apache/fury/serializer/ArraySerializers.java index 75155f0191..a6daec1e27 100644 --- a/java/fury-core/src/main/java/org/apache/fury/serializer/ArraySerializers.java +++ b/java/fury-core/src/main/java/org/apache/fury/serializer/ArraySerializers.java @@ -111,7 +111,7 @@ public void write(MemoryBuffer buffer, T[] arr) { list.setArray(arr); fury.incDepth(1); this.collectionSerializer.write(buffer, list); - fury.incDepth(1); + fury.incDepth(-1); fury.getGenerics().popGenericType(); this.list = list; } else { From 24ee77c19cc603f6cbedfc21fe68164472ecf6af Mon Sep 17 00:00:00 2001 From: David Sherrier Date: Fri, 16 May 2025 22:07:06 +0100 Subject: [PATCH 5/5] fix: Handle primitive casting in read --- .../apache/fury/serializer/ArraySerializers.java | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/java/fury-core/src/main/java/org/apache/fury/serializer/ArraySerializers.java b/java/fury-core/src/main/java/org/apache/fury/serializer/ArraySerializers.java index a6daec1e27..662e480907 100644 --- a/java/fury-core/src/main/java/org/apache/fury/serializer/ArraySerializers.java +++ b/java/fury-core/src/main/java/org/apache/fury/serializer/ArraySerializers.java @@ -101,7 +101,10 @@ public void write(MemoryBuffer buffer, T[] arr) { RefResolver refResolver = fury.getRefResolver(); Serializer componentSerializer = this.componentTypeSerializer; if (this.collectionSerializer != null) { - fury.getGenerics().pushGenericType(this.collectionGenericType); + buffer.writeVarUint32Small7(len); + if (len == 0) { + return; + } ArrayAsList list = this.list; if (list == null) { list = new ArrayAsList(0); @@ -110,9 +113,10 @@ public void write(MemoryBuffer buffer, T[] arr) { } list.setArray(arr); fury.incDepth(1); + fury.getGenerics().pushGenericType(this.collectionGenericType); this.collectionSerializer.write(buffer, list); - fury.incDepth(-1); fury.getGenerics().popGenericType(); + fury.incDepth(-1); this.list = list; } else { int header = componentSerializer != null ? 0b1 : 0b0; @@ -188,12 +192,15 @@ public void xwrite(MemoryBuffer buffer, T[] arr) { @Override public T[] read(MemoryBuffer buffer) { Object[] value; + int numElements = buffer.readVarUint32Small7(); if (this.collectionSerializer != null) { fury.getGenerics().pushGenericType(this.collectionGenericType); value = this.collectionSerializer.read(buffer).toArray(); fury.getGenerics().popGenericType(); + if (this.innerType.isPrimitive() || TypeUtils.isBoxed(this.innerType)) { + return Arrays.copyOf(value, value.length, this.type); + } } else { - int numElements = buffer.readVarUint32Small7(); boolean isFinal = (numElements & 0b1) != 0; numElements >>>= 1; value = newArray(numElements);