From 2c0352480a40c00b3ff0ac2dc2a95817a2f199e4 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sat, 23 May 2026 20:41:21 +0800 Subject: [PATCH 01/69] refactor(java): move buffer equality out of UnsafeOps --- .../org/apache/fory/memory/MemoryBuffer.java | 36 ++++++- .../org/apache/fory/platform/UnsafeOps.java | 41 -------- .../apache/fory/memory/MemoryBufferTest.java | 3 + .../apache/fory/reflect/UnsafeOpsTest.java | 93 ------------------- 4 files changed, 36 insertions(+), 137 deletions(-) delete mode 100644 java/fory-core/src/test/java/org/apache/fory/reflect/UnsafeOpsTest.java diff --git a/java/fory-core/src/main/java/org/apache/fory/memory/MemoryBuffer.java b/java/fory-core/src/main/java/org/apache/fory/memory/MemoryBuffer.java index c6388f1041..89fc343931 100644 --- a/java/fory-core/src/main/java/org/apache/fory/memory/MemoryBuffer.java +++ b/java/fory-core/src/main/java/org/apache/fory/memory/MemoryBuffer.java @@ -64,6 +64,7 @@ public final class MemoryBuffer { public static final int BUFFER_GROW_STEP_THRESHOLD = 100 * 1024 * 1024; private static final Unsafe UNSAFE = AndroidSupport.IS_ANDROID ? null : UnsafeOps.UNSAFE; private static final boolean LITTLE_ENDIAN = NativeByteOrder.IS_LITTLE_ENDIAN; + private static final boolean UNALIGNED = !AndroidSupport.IS_ANDROID && UnsafeOps.unaligned(); // Global allocator instance that can be customized private static volatile MemoryAllocator globalAllocator = new DefaultMemoryAllocator(); @@ -3716,7 +3717,7 @@ public boolean equalTo(MemoryBuffer buf2, int offset1, int offset2, int len) { final long pos2 = buf2.address + offset2; checkArgument(pos1 < addressLimit); checkArgument(pos2 < buf2.addressLimit); - return UnsafeOps.arrayEquals(heapMemory, pos1, buf2.heapMemory, pos2, len); + return unsafeEqualTo(heapMemory, pos1, buf2.heapMemory, pos2, len); } /** @@ -3740,8 +3741,37 @@ public boolean equalTo(byte[] bytes, int bytesOffset, int offset, int len) { return MemoryOps.equalTo(this, bytes, bytesOffset, offset, len); } final long pos = address + offset; - return UnsafeOps.arrayEquals( - heapMemory, pos, bytes, UnsafeOps.BYTE_ARRAY_OFFSET + bytesOffset, len); + return unsafeEqualTo(heapMemory, pos, bytes, UnsafeOps.BYTE_ARRAY_OFFSET + bytesOffset, len); + } + + private static boolean unsafeEqualTo( + Object leftBase, long leftOffset, Object rightBase, long rightOffset, int length) { + int i = 0; + if ((leftOffset % 8) == (rightOffset % 8)) { + while ((leftOffset + i) % 8 != 0 && i < length) { + if (UNSAFE.getByte(leftBase, leftOffset + i) + != UNSAFE.getByte(rightBase, rightOffset + i)) { + return false; + } + i += 1; + } + } + if (UNALIGNED || (((leftOffset + i) % 8 == 0) && ((rightOffset + i) % 8 == 0))) { + while (i <= length - 8) { + if (UNSAFE.getLong(leftBase, leftOffset + i) + != UNSAFE.getLong(rightBase, rightOffset + i)) { + return false; + } + i += 8; + } + } + while (i < length) { + if (UNSAFE.getByte(leftBase, leftOffset + i) != UNSAFE.getByte(rightBase, rightOffset + i)) { + return false; + } + i += 1; + } + return true; } @Override diff --git a/java/fory-core/src/main/java/org/apache/fory/platform/UnsafeOps.java b/java/fory-core/src/main/java/org/apache/fory/platform/UnsafeOps.java index c8ddf1ae5a..6c9c273016 100644 --- a/java/fory-core/src/main/java/org/apache/fory/platform/UnsafeOps.java +++ b/java/fory-core/src/main/java/org/apache/fory/platform/UnsafeOps.java @@ -234,47 +234,6 @@ public static Object[] copyObjectArray(Object[] arr) { return objects; } - /** - * Optimized byte array equality check for byte arrays. - * - * @return true if the arrays are equal, false otherwise - */ - public static boolean arrayEquals( - Object leftBase, long leftOffset, Object rightBase, long rightOffset, final long length) { - int i = 0; - - // check if stars align and we can get both offsets to be aligned - if ((leftOffset % 8) == (rightOffset % 8)) { - while ((leftOffset + i) % 8 != 0 && i < length) { - if (UnsafeOps.getByte(leftBase, leftOffset + i) - != UnsafeOps.getByte(rightBase, rightOffset + i)) { - return false; - } - i += 1; - } - } - // for architectures that support unaligned accesses, chew it up 8 bytes at a time - if (unaligned || (((leftOffset + i) % 8 == 0) && ((rightOffset + i) % 8 == 0))) { - while (i <= length - 8) { - if (UnsafeOps.getLong(leftBase, leftOffset + i) - != UnsafeOps.getLong(rightBase, rightOffset + i)) { - return false; - } - i += 8; - } - } - // this will finish off the unaligned comparisons, or do the entire aligned - // comparison whichever is needed. - while (i < length) { - if (UnsafeOps.getByte(leftBase, leftOffset + i) - != UnsafeOps.getByte(rightBase, rightOffset + i)) { - return false; - } - i += 1; - } - return true; - } - /** Create an instance of type. This method don't call constructor. */ public static T newInstance(Class type) { try { diff --git a/java/fory-core/src/test/java/org/apache/fory/memory/MemoryBufferTest.java b/java/fory-core/src/test/java/org/apache/fory/memory/MemoryBufferTest.java index 0b9e41b79e..1af57bf458 100644 --- a/java/fory-core/src/test/java/org/apache/fory/memory/MemoryBufferTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/memory/MemoryBufferTest.java @@ -340,8 +340,11 @@ public void testEqualTo() { buf1.putByte(9, (byte) 1); buf2.putByte(9, (byte) 1); Assert.assertTrue(buf1.equalTo(buf2, 0, 0, buf1.size())); + Assert.assertTrue(buf1.equalTo(buf2, 1, 1, 9)); + Assert.assertTrue(buf1.equalTo(new byte[] {0, 0, 0, 0, 0, 0, 0, 0, 1}, 0, 1, 9)); buf1.putByte(9, (byte) 2); Assert.assertFalse(buf1.equalTo(buf2, 0, 0, buf1.size())); + Assert.assertFalse(buf1.equalTo(buf2, 1, 1, 9)); } @Test diff --git a/java/fory-core/src/test/java/org/apache/fory/reflect/UnsafeOpsTest.java b/java/fory-core/src/test/java/org/apache/fory/reflect/UnsafeOpsTest.java deleted file mode 100644 index 804cf6a86b..0000000000 --- a/java/fory-core/src/test/java/org/apache/fory/reflect/UnsafeOpsTest.java +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.fory.reflect; - -import static org.testng.Assert.assertTrue; - -import java.nio.charset.StandardCharsets; -import java.util.Arrays; -import org.apache.fory.platform.UnsafeOps; -import org.testng.annotations.Test; - -public class UnsafeOpsTest { - @Test - public void testArrayEquals() { - byte[] bytes = "123456781234567".getBytes(StandardCharsets.UTF_8); - byte[] bytes2 = "123456781234567".getBytes(StandardCharsets.UTF_8); - assert bytes.length == bytes2.length; - assertTrue( - UnsafeOps.arrayEquals( - bytes, UnsafeOps.BYTE_ARRAY_OFFSET, bytes2, UnsafeOps.BYTE_ARRAY_OFFSET, bytes.length)); - } - - @Test(enabled = false) - public void benchmarkArrayEquals() { - byte[] bytes = "123456781234567".getBytes(StandardCharsets.UTF_8); - byte[] bytes2 = "123456781234567".getBytes(StandardCharsets.UTF_8); - arrayEquals(bytes, bytes2); - bytes = "1234567812345678".getBytes(StandardCharsets.UTF_8); - bytes2 = "1234567812345678".getBytes(StandardCharsets.UTF_8); - arrayEquals(bytes, bytes2); - } - - private boolean arrayEquals(byte[] bytes, byte[] bytes2) { - long nums = 200_000_000; - boolean eq = false; - { - // warm - for (int i = 0; i < nums; i++) { - eq = - bytes.length == bytes2.length - && UnsafeOps.arrayEquals( - bytes, - UnsafeOps.BYTE_ARRAY_OFFSET, - bytes2, - UnsafeOps.BYTE_ARRAY_OFFSET, - bytes.length); - } - long t = System.nanoTime(); - for (int i = 0; i < nums; i++) { - eq = - bytes.length == bytes2.length - && UnsafeOps.arrayEquals( - bytes, - UnsafeOps.BYTE_ARRAY_OFFSET, - bytes2, - UnsafeOps.BYTE_ARRAY_OFFSET, - bytes.length); - } - long duration = System.nanoTime() - t; - System.out.format("native cost %sns %sms\n", duration, duration / 1000_000); - } - { - // warm - for (int i = 0; i < nums; i++) { - eq = Arrays.equals(bytes, bytes2); - } - long t = System.nanoTime(); - for (int i = 0; i < nums; i++) { - eq = Arrays.equals(bytes, bytes2); - } - long duration = System.nanoTime() - t; - System.out.format("Arrays.equals cost %sns %sms\n", duration, duration / 1000_000); - } - return eq; - } -} From b9129acaf7cf330f08dc1bf714a359dac25595fc Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sat, 23 May 2026 20:53:07 +0800 Subject: [PATCH 02/69] refactor(java): move direct buffer address access to MemoryBuffer --- .../apache/fory/memory/ByteBufferUtil.java | 27 ------------- .../org/apache/fory/memory/MemoryBuffer.java | 40 +++++++++++++++---- 2 files changed, 33 insertions(+), 34 deletions(-) diff --git a/java/fory-core/src/main/java/org/apache/fory/memory/ByteBufferUtil.java b/java/fory-core/src/main/java/org/apache/fory/memory/ByteBufferUtil.java index 372699b8be..c43fc255e7 100644 --- a/java/fory-core/src/main/java/org/apache/fory/memory/ByteBufferUtil.java +++ b/java/fory-core/src/main/java/org/apache/fory/memory/ByteBufferUtil.java @@ -19,40 +19,13 @@ package org.apache.fory.memory; -import java.lang.reflect.Field; import java.nio.Buffer; import java.nio.ByteBuffer; -import org.apache.fory.platform.UnsafeOps; -import org.apache.fory.util.Preconditions; public class ByteBufferUtil { public static final Class HEAP_BYTE_BUFFER_CLASS = ByteBuffer.allocate(0).getClass(); public static final Class DIRECT_BYTE_BUFFER_CLASS = ByteBuffer.allocateDirect(0).getClass(); - private static final class DirectBufferAccess { - private static final long BUFFER_ADDRESS_FIELD_OFFSET; - - static { - try { - Field addressField = Buffer.class.getDeclaredField("address"); - BUFFER_ADDRESS_FIELD_OFFSET = UnsafeOps.objectFieldOffset(addressField); - Preconditions.checkArgument(BUFFER_ADDRESS_FIELD_OFFSET != 0); - } catch (NoSuchFieldException e) { - throw new IllegalStateException(e); - } - } - } - - static long getAddress(ByteBuffer buffer) { - Preconditions.checkNotNull(buffer, "buffer is null"); - Preconditions.checkArgument(buffer.isDirect(), "Can't get address of a non-direct ByteBuffer."); - try { - return UnsafeOps.getLong(buffer, DirectBufferAccess.BUFFER_ADDRESS_FIELD_OFFSET); - } catch (Throwable t) { - throw new Error("Could not access direct byte buffer address field.", t); - } - } - public static void clearBuffer(Buffer buffer) { buffer.clear(); } diff --git a/java/fory-core/src/main/java/org/apache/fory/memory/MemoryBuffer.java b/java/fory-core/src/main/java/org/apache/fory/memory/MemoryBuffer.java index 89fc343931..47d69b6ee2 100644 --- a/java/fory-core/src/main/java/org/apache/fory/memory/MemoryBuffer.java +++ b/java/fory-core/src/main/java/org/apache/fory/memory/MemoryBuffer.java @@ -19,7 +19,10 @@ package org.apache.fory.memory; import static org.apache.fory.util.Preconditions.checkArgument; +import static org.apache.fory.util.Preconditions.checkNotNull; +import java.lang.reflect.Field; +import java.nio.Buffer; import java.nio.ByteBuffer; import java.util.Arrays; import org.apache.fory.annotation.CodegenInvoke; @@ -68,6 +71,20 @@ public final class MemoryBuffer { // Global allocator instance that can be customized private static volatile MemoryAllocator globalAllocator = new DefaultMemoryAllocator(); + private static final class DirectBufferAccess { + private static final long BUFFER_ADDRESS_FIELD_OFFSET; + + static { + try { + Field addressField = Buffer.class.getDeclaredField("address"); + BUFFER_ADDRESS_FIELD_OFFSET = UnsafeOps.objectFieldOffset(addressField); + checkArgument(BUFFER_ADDRESS_FIELD_OFFSET != 0); + } catch (NoSuchFieldException e) { + throw new IllegalStateException(e); + } + } + } + // If the data in on the heap, `heapMemory` will be non-null, and its' the object relative to // which we access the memory. // If we have this buffer, we must never void this reference, or the memory buffer will point @@ -187,12 +204,22 @@ private void initOffHeapBuffer(long offHeapAddress, int size, ByteBuffer offHeap this.size = size; } + private static long getAddress(ByteBuffer buffer) { + checkNotNull(buffer, "buffer is null"); + checkArgument(buffer.isDirect(), "Can't get address of a non-direct ByteBuffer."); + try { + return UnsafeOps.getLong(buffer, DirectBufferAccess.BUFFER_ADDRESS_FIELD_OFFSET); + } catch (Throwable t) { + throw new Error("Could not access direct byte buffer address field.", t); + } + } + public void initByteBuffer(ByteBuffer buffer, int size) { if (buffer.isDirect()) { if (AndroidSupport.IS_ANDROID) { MemoryOps.throwDirectByteBufferUnsupported(); } else { - initOffHeapBuffer(ByteBufferUtil.getAddress(buffer), size, buffer); + initOffHeapBuffer(getAddress(buffer), size, buffer); } } else if (buffer.hasArray()) { initHeapBuffer(buffer.array(), buffer.arrayOffset(), size); @@ -370,7 +397,7 @@ public void get(int offset, ByteBuffer target, int numBytes) { if (AndroidSupport.IS_ANDROID) { MemoryOps.get(this, offset, target, numBytes); } else if (target.isDirect()) { - final long targetAddr = ByteBufferUtil.getAddress(target) + targetPos; + final long targetAddr = getAddress(target) + targetPos; final long sourceAddr = address + offset; if (sourceAddr <= addressLimit - numBytes) { UnsafeOps.copyMemory(heapMemory, sourceAddr, null, targetAddr, numBytes); @@ -395,7 +422,7 @@ public void put(int offset, ByteBuffer source, int numBytes) { if (AndroidSupport.IS_ANDROID) { MemoryOps.put(this, offset, source, numBytes); } else if (source.isDirect()) { - final long sourceAddr = ByteBufferUtil.getAddress(source) + sourcePos; + final long sourceAddr = getAddress(source) + sourcePos; final long targetAddr = address + offset; if (targetAddr <= addressLimit - numBytes) { UnsafeOps.copyMemory(null, sourceAddr, heapMemory, targetAddr, numBytes); @@ -3678,7 +3705,7 @@ public ByteBuffer sliceAsByteBuffer(int offset, int length) { ByteBuffer offHeapBuffer = this.offHeapBuffer; if (offHeapBuffer != null) { ByteBuffer duplicate = offHeapBuffer.duplicate(); - int start = (int) (address - ByteBufferUtil.getAddress(duplicate)); + int start = (int) (address - getAddress(duplicate)); ByteBufferUtil.position(duplicate, start + offset); duplicate.limit(start + offset + length); return duplicate.slice(); @@ -3878,8 +3905,7 @@ public static MemoryBuffer fromByteBuffer(ByteBuffer buffer) { if (AndroidSupport.IS_ANDROID) { return MemoryOps.fromByteBuffer(buffer); } else if (buffer.isDirect()) { - return new MemoryBuffer( - ByteBufferUtil.getAddress(buffer) + buffer.position(), buffer.remaining(), buffer); + return new MemoryBuffer(getAddress(buffer) + buffer.position(), buffer.remaining(), buffer); } else if (buffer.hasArray()) { int offset = buffer.arrayOffset() + buffer.position(); return new MemoryBuffer(buffer.array(), offset, buffer.remaining()); @@ -3896,7 +3922,7 @@ public static MemoryBuffer fromDirectByteBuffer( if (AndroidSupport.IS_ANDROID) { return MemoryOps.directByteBufferUnsupported(); } - long offHeapAddress = ByteBufferUtil.getAddress(buffer) + buffer.position(); + long offHeapAddress = getAddress(buffer) + buffer.position(); return new MemoryBuffer(offHeapAddress, size, buffer, streamReader); } From 1fcdfeab6afeb45884e148ca8b01de9ac02bb20c Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sat, 23 May 2026 20:55:45 +0800 Subject: [PATCH 03/69] remove unused api in UnsafeOps --- .../org/apache/fory/platform/UnsafeOps.java | 37 ------------------- 1 file changed, 37 deletions(-) diff --git a/java/fory-core/src/main/java/org/apache/fory/platform/UnsafeOps.java b/java/fory-core/src/main/java/org/apache/fory/platform/UnsafeOps.java index 6c9c273016..7ca7d09bd0 100644 --- a/java/fory-core/src/main/java/org/apache/fory/platform/UnsafeOps.java +++ b/java/fory-core/src/main/java/org/apache/fory/platform/UnsafeOps.java @@ -182,37 +182,6 @@ public static void putObject(Object object, long offset, Object value) { UNSAFE.putObject(object, offset, value); } - public static Object getObjectVolatile(Object object, long offset) { - return UNSAFE.getObjectVolatile(object, offset); - } - - public static void putObjectVolatile(Object object, long offset, Object value) { - UNSAFE.putObjectVolatile(object, offset, value); - } - - public static long allocateMemory(long size) { - return UNSAFE.allocateMemory(size); - } - - public static void freeMemory(long address) { - UNSAFE.freeMemory(address); - } - - public static long reallocateMemory(long address, long oldSize, long newSize) { - long newMemory = UNSAFE.allocateMemory(newSize); - copyMemory(null, address, null, newMemory, oldSize); - freeMemory(address); - return newMemory; - } - - public static void setMemory(Object object, long offset, long size, byte value) { - UNSAFE.setMemory(object, offset, size, value); - } - - public static void setMemory(long address, byte value, long size) { - UNSAFE.setMemory(address, size, value); - } - public static void copyMemory( Object src, long srcOffset, Object dst, long dstOffset, long length) { if (length < UNSAFE_COPY_THRESHOLD) { @@ -228,12 +197,6 @@ public static void copyMemory( } } - public static Object[] copyObjectArray(Object[] arr) { - Object[] objects = new Object[arr.length]; - System.arraycopy(arr, 0, objects, 0, arr.length); - return objects; - } - /** Create an instance of type. This method don't call constructor. */ public static T newInstance(Class type) { try { From 83e1015cda494b064a166ab8f1519159921e5579 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sat, 23 May 2026 21:15:19 +0800 Subject: [PATCH 04/69] refactor(java): hide field offsets behind accessors --- .../apache/fory/reflect/FieldAccessor.java | 358 ++++++++++++++++- .../serializer/AbstractObjectSerializer.java | 368 ++++-------------- .../apache/fory/util/DefaultValueUtils.java | 20 +- .../fory/reflect/FieldAccessorTest.java | 10 +- 4 files changed, 429 insertions(+), 327 deletions(-) diff --git a/java/fory-core/src/main/java/org/apache/fory/reflect/FieldAccessor.java b/java/fory-core/src/main/java/org/apache/fory/reflect/FieldAccessor.java index 1690fc0918..eda245f973 100644 --- a/java/fory-core/src/main/java/org/apache/fory/reflect/FieldAccessor.java +++ b/java/fory-core/src/main/java/org/apache/fory/reflect/FieldAccessor.java @@ -47,11 +47,7 @@ import org.apache.fory.util.record.RecordUtils; import org.apache.fory.util.unsafe._JDKAccess; -/** - * Field accessor for primitive types and object types. - * - *

Note for primitive types, there will be box/unbox overhead. - */ +/** Field accessor for primitive types and object types. */ @SuppressWarnings({"unchecked", "rawtypes"}) public abstract class FieldAccessor { protected final Field field; @@ -84,7 +80,71 @@ public Field getField() { return field; } - public final void putObject(Object targetObject, Object object) { + public boolean getBoolean(Object targetObject) { + return (Boolean) get(targetObject); + } + + public void putBoolean(Object targetObject, boolean value) { + set(targetObject, value); + } + + public byte getByte(Object targetObject) { + return (Byte) get(targetObject); + } + + public void putByte(Object targetObject, byte value) { + set(targetObject, value); + } + + public char getChar(Object targetObject) { + return (Character) get(targetObject); + } + + public void putChar(Object targetObject, char value) { + set(targetObject, value); + } + + public short getShort(Object targetObject) { + return (Short) get(targetObject); + } + + public void putShort(Object targetObject, short value) { + set(targetObject, value); + } + + public int getInt(Object targetObject) { + return (Integer) get(targetObject); + } + + public void putInt(Object targetObject, int value) { + set(targetObject, value); + } + + public long getLong(Object targetObject) { + return (Long) get(targetObject); + } + + public void putLong(Object targetObject, long value) { + set(targetObject, value); + } + + public float getFloat(Object targetObject) { + return (Float) get(targetObject); + } + + public void putFloat(Object targetObject, float value) { + set(targetObject, value); + } + + public double getDouble(Object targetObject) { + return (Double) get(targetObject); + } + + public void putDouble(Object targetObject, double value) { + set(targetObject, value); + } + + public void putObject(Object targetObject, Object object) { // For primitive fields, we must use set() which calls the correct UnsafeOps.putXxx method. // UnsafeOps.putObject writes object references, not primitive values. if (fieldOffset != -1 && !field.getType().isPrimitive()) { @@ -94,7 +154,7 @@ public final void putObject(Object targetObject, Object object) { } } - public final Object getObject(Object targetObject) { + public Object getObject(Object targetObject) { // For primitive fields, we must use get() which calls the correct UnsafeOps.getXxx method // and returns the boxed value. UnsafeOps.getObject interprets primitive bytes as object // refs. @@ -105,10 +165,6 @@ public final Object getObject(Object targetObject) { } } - public long getFieldOffset() { - return fieldOffset; - } - void checkObj(Object obj) { if (!this.field.getDeclaringClass().isAssignableFrom(obj.getClass())) { throw new IllegalArgumentException("Illegal class " + obj.getClass()); @@ -236,14 +292,24 @@ public BooleanAccessor(Field field) { @Override public Object get(Object obj) { + return getBoolean(obj); + } + + @Override + public boolean getBoolean(Object obj) { checkObj(obj); return UnsafeOps.getBoolean(obj, fieldOffset); } @Override public void set(Object obj, Object value) { + putBoolean(obj, (Boolean) value); + } + + @Override + public void putBoolean(Object obj, boolean value) { checkObj(obj); - UnsafeOps.putBoolean(obj, fieldOffset, (Boolean) value); + UnsafeOps.putBoolean(obj, fieldOffset, value); } } @@ -258,6 +324,11 @@ public BooleanGetter(Field field, Predicate getter) { @Override public Boolean get(Object obj) { + return getBoolean(obj); + } + + @Override + public boolean getBoolean(Object obj) { checkObj(obj); return getter.test(obj); } @@ -272,14 +343,24 @@ public ByteAccessor(Field field) { @Override public Byte get(Object obj) { + return getByte(obj); + } + + @Override + public byte getByte(Object obj) { checkObj(obj); return UnsafeOps.getByte(obj, fieldOffset); } @Override public void set(Object obj, Object value) { + putByte(obj, (Byte) value); + } + + @Override + public void putByte(Object obj, byte value) { checkObj(obj); - UnsafeOps.putByte(obj, fieldOffset, (Byte) value); + UnsafeOps.putByte(obj, fieldOffset, value); } } @@ -295,6 +376,11 @@ public ByteGetter(Field field, ToByteFunction getter) { @Override public Byte get(Object obj) { + return getByte(obj); + } + + @Override + public byte getByte(Object obj) { return getter.applyAsByte(obj); } } @@ -308,14 +394,24 @@ public CharAccessor(Field field) { @Override public Character get(Object obj) { + return getChar(obj); + } + + @Override + public char getChar(Object obj) { checkObj(obj); return UnsafeOps.getChar(obj, fieldOffset); } @Override public void set(Object obj, Object value) { + putChar(obj, (Character) value); + } + + @Override + public void putChar(Object obj, char value) { checkObj(obj); - UnsafeOps.putChar(obj, fieldOffset, (Character) value); + UnsafeOps.putChar(obj, fieldOffset, value); } } @@ -330,6 +426,11 @@ public CharGetter(Field field, ToCharFunction getter) { @Override public Character get(Object obj) { + return getChar(obj); + } + + @Override + public char getChar(Object obj) { return getter.applyAsChar(obj); } } @@ -343,14 +444,24 @@ public ShortAccessor(Field field) { @Override public Short get(Object obj) { + return getShort(obj); + } + + @Override + public short getShort(Object obj) { checkObj(obj); return UnsafeOps.getShort(obj, fieldOffset); } @Override public void set(Object obj, Object value) { + putShort(obj, (Short) value); + } + + @Override + public void putShort(Object obj, short value) { checkObj(obj); - UnsafeOps.putShort(obj, fieldOffset, (Short) value); + UnsafeOps.putShort(obj, fieldOffset, value); } } @@ -365,6 +476,11 @@ public ShortGetter(Field field, ToShortFunction getter) { @Override public Short get(Object obj) { + return getShort(obj); + } + + @Override + public short getShort(Object obj) { return getter.applyAsShort(obj); } } @@ -378,14 +494,24 @@ public IntAccessor(Field field) { @Override public Integer get(Object obj) { + return getInt(obj); + } + + @Override + public int getInt(Object obj) { checkObj(obj); return UnsafeOps.getInt(obj, fieldOffset); } @Override public void set(Object obj, Object value) { + putInt(obj, (Integer) value); + } + + @Override + public void putInt(Object obj, int value) { checkObj(obj); - UnsafeOps.putInt(obj, fieldOffset, (Integer) value); + UnsafeOps.putInt(obj, fieldOffset, value); } } @@ -400,6 +526,11 @@ public IntGetter(Field field, ToIntFunction getter) { @Override public Integer get(Object obj) { + return getInt(obj); + } + + @Override + public int getInt(Object obj) { return getter.applyAsInt(obj); } } @@ -413,14 +544,24 @@ public LongAccessor(Field field) { @Override public Long get(Object obj) { + return getLong(obj); + } + + @Override + public long getLong(Object obj) { checkObj(obj); return UnsafeOps.getLong(obj, fieldOffset); } @Override public void set(Object obj, Object value) { + putLong(obj, (Long) value); + } + + @Override + public void putLong(Object obj, long value) { checkObj(obj); - UnsafeOps.putLong(obj, fieldOffset, (Long) value); + UnsafeOps.putLong(obj, fieldOffset, value); } } @@ -435,6 +576,11 @@ public LongGetter(Field field, ToLongFunction getter) { @Override public Long get(Object obj) { + return getLong(obj); + } + + @Override + public long getLong(Object obj) { return getter.applyAsLong(obj); } } @@ -448,14 +594,24 @@ public FloatAccessor(Field field) { @Override public Object get(Object obj) { + return getFloat(obj); + } + + @Override + public float getFloat(Object obj) { checkObj(obj); return UnsafeOps.getFloat(obj, fieldOffset); } @Override public void set(Object obj, Object value) { + putFloat(obj, (Float) value); + } + + @Override + public void putFloat(Object obj, float value) { checkObj(obj); - UnsafeOps.putFloat(obj, fieldOffset, (Float) value); + UnsafeOps.putFloat(obj, fieldOffset, value); } } @@ -470,6 +626,11 @@ public FloatGetter(Field field, ToFloatFunction getter) { @Override public Float get(Object obj) { + return getFloat(obj); + } + + @Override + public float getFloat(Object obj) { return getter.applyAsFloat(obj); } } @@ -483,14 +644,24 @@ public DoubleAccessor(Field field) { @Override public Object get(Object obj) { + return getDouble(obj); + } + + @Override + public double getDouble(Object obj) { checkObj(obj); return UnsafeOps.getDouble(obj, fieldOffset); } @Override public void set(Object obj, Object value) { + putDouble(obj, (Double) value); + } + + @Override + public void putDouble(Object obj, double value) { checkObj(obj); - UnsafeOps.putDouble(obj, fieldOffset, (Double) value); + UnsafeOps.putDouble(obj, fieldOffset, value); } } @@ -505,6 +676,11 @@ public DoubleGetter(Field field, ToDoubleFunction getter) { @Override public Double get(Object obj) { + return getDouble(obj); + } + + @Override + public double getDouble(Object obj) { return getter.applyAsDouble(obj); } } @@ -594,5 +770,149 @@ public void set(Object obj, Object value) { throw new RuntimeException(e); } } + + @Override + public boolean getBoolean(Object obj) { + try { + return (boolean) getter.invoke(obj); + } catch (Throwable e) { + throw new RuntimeException(e); + } + } + + @Override + public void putBoolean(Object obj, boolean value) { + try { + setter.invoke(obj, value); + } catch (Throwable e) { + throw new RuntimeException(e); + } + } + + @Override + public byte getByte(Object obj) { + try { + return (byte) getter.invoke(obj); + } catch (Throwable e) { + throw new RuntimeException(e); + } + } + + @Override + public void putByte(Object obj, byte value) { + try { + setter.invoke(obj, value); + } catch (Throwable e) { + throw new RuntimeException(e); + } + } + + @Override + public char getChar(Object obj) { + try { + return (char) getter.invoke(obj); + } catch (Throwable e) { + throw new RuntimeException(e); + } + } + + @Override + public void putChar(Object obj, char value) { + try { + setter.invoke(obj, value); + } catch (Throwable e) { + throw new RuntimeException(e); + } + } + + @Override + public short getShort(Object obj) { + try { + return (short) getter.invoke(obj); + } catch (Throwable e) { + throw new RuntimeException(e); + } + } + + @Override + public void putShort(Object obj, short value) { + try { + setter.invoke(obj, value); + } catch (Throwable e) { + throw new RuntimeException(e); + } + } + + @Override + public int getInt(Object obj) { + try { + return (int) getter.invoke(obj); + } catch (Throwable e) { + throw new RuntimeException(e); + } + } + + @Override + public void putInt(Object obj, int value) { + try { + setter.invoke(obj, value); + } catch (Throwable e) { + throw new RuntimeException(e); + } + } + + @Override + public long getLong(Object obj) { + try { + return (long) getter.invoke(obj); + } catch (Throwable e) { + throw new RuntimeException(e); + } + } + + @Override + public void putLong(Object obj, long value) { + try { + setter.invoke(obj, value); + } catch (Throwable e) { + throw new RuntimeException(e); + } + } + + @Override + public float getFloat(Object obj) { + try { + return (float) getter.invoke(obj); + } catch (Throwable e) { + throw new RuntimeException(e); + } + } + + @Override + public void putFloat(Object obj, float value) { + try { + setter.invoke(obj, value); + } catch (Throwable e) { + throw new RuntimeException(e); + } + } + + @Override + public double getDouble(Object obj) { + try { + return (double) getter.invoke(obj); + } catch (Throwable e) { + throw new RuntimeException(e); + } + } + + @Override + public void putDouble(Object obj, double value) { + try { + setter.invoke(obj, value); + } catch (Throwable e) { + throw new RuntimeException(e); + } + } } } diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/AbstractObjectSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/AbstractObjectSerializer.java index 4ac9e2ab5f..58bc955f9d 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/AbstractObjectSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/AbstractObjectSerializer.java @@ -35,7 +35,6 @@ import org.apache.fory.logging.Logger; import org.apache.fory.logging.LoggerFactory; import org.apache.fory.memory.MemoryBuffer; -import org.apache.fory.platform.UnsafeOps; import org.apache.fory.reflect.FieldAccessor; import org.apache.fory.reflect.ObjectCreator; import org.apache.fory.reflect.ObjectCreators; @@ -243,75 +242,6 @@ static void writeBuildInFieldValue( } } - /** - * Write a primitive field value to buffer using direct memory offset access. - * - * @param buffer the buffer to write to - * @param targetObject the object containing the field - * @param fieldOffset the memory offset of the field - * @param dispatchId the class ID of the primitive type - * @return true if dispatchId is not a primitive type and needs further write handling - */ - private static boolean writePrimitiveFieldValue( - MemoryBuffer buffer, Object targetObject, long fieldOffset, int dispatchId) { - switch (dispatchId) { - case DispatchId.BOOL: - buffer.writeBoolean(UnsafeOps.getBoolean(targetObject, fieldOffset)); - return false; - case DispatchId.INT8: - buffer.writeByte(UnsafeOps.getByte(targetObject, fieldOffset)); - return false; - case DispatchId.UINT8: - buffer.writeByte(UnsafeOps.getInt(targetObject, fieldOffset)); - return false; - case DispatchId.CHAR: - buffer.writeChar(UnsafeOps.getChar(targetObject, fieldOffset)); - return false; - case DispatchId.INT16: - buffer.writeInt16(UnsafeOps.getShort(targetObject, fieldOffset)); - return false; - case DispatchId.UINT16: - buffer.writeInt16((short) UnsafeOps.getInt(targetObject, fieldOffset)); - return false; - case DispatchId.INT32: - buffer.writeInt32(UnsafeOps.getInt(targetObject, fieldOffset)); - return false; - case DispatchId.UINT32: - buffer.writeInt32((int) UnsafeOps.getLong(targetObject, fieldOffset)); - return false; - case DispatchId.VARINT32: - buffer.writeVarInt32(UnsafeOps.getInt(targetObject, fieldOffset)); - return false; - case DispatchId.VAR_UINT32: - buffer.writeVarUInt32((int) UnsafeOps.getLong(targetObject, fieldOffset)); - return false; - case DispatchId.FLOAT32: - buffer.writeFloat32(UnsafeOps.getFloat(targetObject, fieldOffset)); - return false; - case DispatchId.INT64: - case DispatchId.UINT64: - buffer.writeInt64(UnsafeOps.getLong(targetObject, fieldOffset)); - return false; - case DispatchId.VARINT64: - buffer.writeVarInt64(UnsafeOps.getLong(targetObject, fieldOffset)); - return false; - case DispatchId.TAGGED_INT64: - buffer.writeTaggedInt64(UnsafeOps.getLong(targetObject, fieldOffset)); - return false; - case DispatchId.VAR_UINT64: - buffer.writeVarUInt64(UnsafeOps.getLong(targetObject, fieldOffset)); - return false; - case DispatchId.TAGGED_UINT64: - buffer.writeTaggedUInt64(UnsafeOps.getLong(targetObject, fieldOffset)); - return false; - case DispatchId.FLOAT64: - buffer.writeFloat64(UnsafeOps.getDouble(targetObject, fieldOffset)); - return false; - default: - return true; - } - } - /** * Write a primitive field value to buffer using the field accessor. * @@ -323,65 +253,58 @@ private static boolean writePrimitiveFieldValue( */ static boolean writePrimitiveFieldValue( MemoryBuffer buffer, Object targetObject, FieldAccessor fieldAccessor, int dispatchId) { - long fieldOffset = fieldAccessor.getFieldOffset(); - if (fieldOffset != -1) { - return writePrimitiveFieldValue(buffer, targetObject, fieldOffset, dispatchId); - } - // graalvm use GeneratedAccessor, which will be this code path. switch (dispatchId) { case DispatchId.BOOL: - buffer.writeBoolean((Boolean) fieldAccessor.get(targetObject)); + buffer.writeBoolean(fieldAccessor.getBoolean(targetObject)); return false; case DispatchId.INT8: - buffer.writeByte((Byte) fieldAccessor.get(targetObject)); + buffer.writeByte(fieldAccessor.getByte(targetObject)); return false; case DispatchId.UINT8: - buffer.writeByte((Integer) fieldAccessor.get(targetObject)); + buffer.writeByte(fieldAccessor.getInt(targetObject)); return false; case DispatchId.CHAR: - buffer.writeChar((Character) fieldAccessor.get(targetObject)); + buffer.writeChar(fieldAccessor.getChar(targetObject)); return false; case DispatchId.INT16: - buffer.writeInt16((Short) fieldAccessor.get(targetObject)); + buffer.writeInt16(fieldAccessor.getShort(targetObject)); return false; case DispatchId.UINT16: - buffer.writeInt16(((Integer) fieldAccessor.get(targetObject)).shortValue()); + buffer.writeInt16((short) fieldAccessor.getInt(targetObject)); return false; case DispatchId.INT32: - buffer.writeInt32((Integer) fieldAccessor.get(targetObject)); + buffer.writeInt32(fieldAccessor.getInt(targetObject)); return false; case DispatchId.UINT32: - buffer.writeInt32(((Long) fieldAccessor.get(targetObject)).intValue()); + buffer.writeInt32((int) fieldAccessor.getLong(targetObject)); return false; case DispatchId.VARINT32: - buffer.writeVarInt32((Integer) fieldAccessor.get(targetObject)); + buffer.writeVarInt32(fieldAccessor.getInt(targetObject)); return false; case DispatchId.VAR_UINT32: - buffer.writeVarUInt32(((Long) fieldAccessor.get(targetObject)).intValue()); + buffer.writeVarUInt32((int) fieldAccessor.getLong(targetObject)); return false; case DispatchId.FLOAT32: - buffer.writeFloat32((Float) fieldAccessor.get(targetObject)); + buffer.writeFloat32(fieldAccessor.getFloat(targetObject)); return false; case DispatchId.INT64: - buffer.writeInt64((Long) fieldAccessor.get(targetObject)); - return false; case DispatchId.UINT64: - buffer.writeInt64((Long) fieldAccessor.get(targetObject)); + buffer.writeInt64(fieldAccessor.getLong(targetObject)); return false; case DispatchId.VARINT64: - buffer.writeVarInt64((Long) fieldAccessor.get(targetObject)); + buffer.writeVarInt64(fieldAccessor.getLong(targetObject)); return false; case DispatchId.TAGGED_INT64: - buffer.writeTaggedInt64((Long) fieldAccessor.get(targetObject)); + buffer.writeTaggedInt64(fieldAccessor.getLong(targetObject)); return false; case DispatchId.VAR_UINT64: - buffer.writeVarUInt64((Long) fieldAccessor.get(targetObject)); + buffer.writeVarUInt64(fieldAccessor.getLong(targetObject)); return false; case DispatchId.TAGGED_UINT64: - buffer.writeTaggedUInt64((Long) fieldAccessor.get(targetObject)); + buffer.writeTaggedUInt64(fieldAccessor.getLong(targetObject)); return false; case DispatchId.FLOAT64: - buffer.writeFloat64((Double) fieldAccessor.get(targetObject)); + buffer.writeFloat64(fieldAccessor.getDouble(targetObject)); return false; default: return true; @@ -770,134 +693,59 @@ private static Object readNotNullBuildInFieldValue( */ private static void readPrimitiveFieldValue( MemoryBuffer buffer, Object targetObject, FieldAccessor fieldAccessor, int dispatchId) { - long fieldOffset = fieldAccessor.getFieldOffset(); - if (fieldOffset != -1) { - readPrimitiveFieldValue(buffer, targetObject, fieldOffset, dispatchId); - return; - } - // graalvm use GeneratedAccessor, which will be this code path. // we still need `PRIMITIVE` cases since peer may send switch (dispatchId) { case DispatchId.BOOL: - fieldAccessor.set(targetObject, buffer.readBoolean()); - return; - case DispatchId.INT8: - fieldAccessor.set(targetObject, buffer.readByte()); - return; - case DispatchId.UINT8: - fieldAccessor.set(targetObject, buffer.readByte() & 0xFF); - return; - case DispatchId.CHAR: - fieldAccessor.set(targetObject, buffer.readChar()); - return; - case DispatchId.INT16: - fieldAccessor.set(targetObject, buffer.readInt16()); - return; - case DispatchId.UINT16: - fieldAccessor.set(targetObject, buffer.readInt16() & 0xFFFF); - return; - case DispatchId.INT32: - fieldAccessor.set(targetObject, buffer.readInt32()); - return; - case DispatchId.UINT32: - fieldAccessor.set(targetObject, Integer.toUnsignedLong(buffer.readInt32())); - return; - case DispatchId.VARINT32: - fieldAccessor.set(targetObject, buffer.readVarInt32()); - return; - case DispatchId.VAR_UINT32: - fieldAccessor.set(targetObject, Integer.toUnsignedLong(buffer.readVarUInt32())); - return; - case DispatchId.FLOAT32: - fieldAccessor.set(targetObject, buffer.readFloat32()); - return; - case DispatchId.INT64: - case DispatchId.UINT64: - fieldAccessor.set(targetObject, buffer.readInt64()); - return; - case DispatchId.VARINT64: - fieldAccessor.set(targetObject, buffer.readVarInt64()); - return; - case DispatchId.TAGGED_INT64: - fieldAccessor.set(targetObject, buffer.readTaggedInt64()); - return; - case DispatchId.VAR_UINT64: - fieldAccessor.set(targetObject, buffer.readVarUInt64()); - return; - case DispatchId.TAGGED_UINT64: - fieldAccessor.set(targetObject, buffer.readTaggedUInt64()); - return; - case DispatchId.FLOAT64: - fieldAccessor.set(targetObject, buffer.readFloat64()); - return; - default: - throw new IllegalArgumentException("Unsupported dispatch id " + dispatchId); - } - } - - /** - * Read a primitive field value from buffer and set it using direct memory offset access. - * - * @param buffer the buffer to read from - * @param targetObject the object to set the field value on - * @param fieldOffset the memory offset of the field - * @param dispatchId the dispatch ID of the primitive type - */ - private static void readPrimitiveFieldValue( - MemoryBuffer buffer, Object targetObject, long fieldOffset, int dispatchId) { - switch (dispatchId) { - case DispatchId.BOOL: - UnsafeOps.putBoolean(targetObject, fieldOffset, buffer.readBoolean()); + fieldAccessor.putBoolean(targetObject, buffer.readBoolean()); return; case DispatchId.INT8: - UnsafeOps.putByte(targetObject, fieldOffset, buffer.readByte()); + fieldAccessor.putByte(targetObject, buffer.readByte()); return; case DispatchId.UINT8: - UnsafeOps.putInt(targetObject, fieldOffset, buffer.readByte() & 0xFF); + fieldAccessor.putInt(targetObject, buffer.readByte() & 0xFF); return; case DispatchId.CHAR: - UnsafeOps.putChar(targetObject, fieldOffset, buffer.readChar()); + fieldAccessor.putChar(targetObject, buffer.readChar()); return; case DispatchId.INT16: - UnsafeOps.putShort(targetObject, fieldOffset, buffer.readInt16()); + fieldAccessor.putShort(targetObject, buffer.readInt16()); return; case DispatchId.UINT16: - UnsafeOps.putInt(targetObject, fieldOffset, buffer.readInt16() & 0xFFFF); + fieldAccessor.putInt(targetObject, buffer.readInt16() & 0xFFFF); return; case DispatchId.INT32: - UnsafeOps.putInt(targetObject, fieldOffset, buffer.readInt32()); + fieldAccessor.putInt(targetObject, buffer.readInt32()); return; case DispatchId.UINT32: - UnsafeOps.putLong(targetObject, fieldOffset, Integer.toUnsignedLong(buffer.readInt32())); + fieldAccessor.putLong(targetObject, Integer.toUnsignedLong(buffer.readInt32())); return; case DispatchId.VARINT32: - UnsafeOps.putInt(targetObject, fieldOffset, buffer.readVarInt32()); + fieldAccessor.putInt(targetObject, buffer.readVarInt32()); return; case DispatchId.VAR_UINT32: - UnsafeOps.putLong( - targetObject, fieldOffset, Integer.toUnsignedLong(buffer.readVarUInt32())); + fieldAccessor.putLong(targetObject, Integer.toUnsignedLong(buffer.readVarUInt32())); return; case DispatchId.FLOAT32: - UnsafeOps.putFloat(targetObject, fieldOffset, buffer.readFloat32()); + fieldAccessor.putFloat(targetObject, buffer.readFloat32()); return; case DispatchId.INT64: case DispatchId.UINT64: - UnsafeOps.putLong(targetObject, fieldOffset, buffer.readInt64()); + fieldAccessor.putLong(targetObject, buffer.readInt64()); return; case DispatchId.VARINT64: - UnsafeOps.putLong(targetObject, fieldOffset, buffer.readVarInt64()); + fieldAccessor.putLong(targetObject, buffer.readVarInt64()); return; case DispatchId.TAGGED_INT64: - UnsafeOps.putLong(targetObject, fieldOffset, buffer.readTaggedInt64()); + fieldAccessor.putLong(targetObject, buffer.readTaggedInt64()); return; case DispatchId.VAR_UINT64: - UnsafeOps.putLong(targetObject, fieldOffset, buffer.readVarUInt64()); + fieldAccessor.putLong(targetObject, buffer.readVarUInt64()); return; case DispatchId.TAGGED_UINT64: - UnsafeOps.putLong(targetObject, fieldOffset, buffer.readTaggedUInt64()); + fieldAccessor.putLong(targetObject, buffer.readTaggedUInt64()); return; case DispatchId.FLOAT64: - UnsafeOps.putDouble(targetObject, fieldOffset, buffer.readFloat64()); + fieldAccessor.putDouble(targetObject, buffer.readFloat64()); return; default: throw new IllegalArgumentException("Unsupported dispatch id " + dispatchId); @@ -1043,18 +891,11 @@ private Object[] copyFields(CopyContext copyContext, T originObj) { for (int i = 0; i < fieldInfos.length; i++) { SerializationFieldInfo fieldInfo = fieldInfos[i]; FieldAccessor fieldAccessor = fieldInfo.fieldAccessor; - long fieldOffset = fieldAccessor.getFieldOffset(); - if (fieldOffset != -1) { - if (fieldInfo.isPrimitiveField) { - fieldValues[i] = copyPrimitiveField(originObj, fieldOffset, fieldInfo.dispatchId); - } else { - fieldValues[i] = - copyNotPrimitiveField(copyContext, originObj, fieldOffset, fieldInfo.dispatchId); - } + if (fieldInfo.isPrimitiveField) { + fieldValues[i] = copyPrimitiveField(originObj, fieldAccessor, fieldInfo.dispatchId); } else { - // field in record class has offset -1 - Object fieldValue = fieldAccessor.get(originObj); - fieldValues[i] = copyContext.copyObject(fieldValue, fieldInfo.dispatchId); + fieldValues[i] = + copyNotPrimitiveField(copyContext, originObj, fieldAccessor, fieldInfo.dispatchId); } } return RecordUtils.remapping(copyRecordInfo, fieldValues); @@ -1075,17 +916,11 @@ public static void copyFields( Object newObj) { for (SerializationFieldInfo fieldInfo : fieldInfos) { FieldAccessor fieldAccessor = fieldInfo.fieldAccessor; - long fieldOffset = fieldAccessor.getFieldOffset(); - if (fieldOffset == -1) { - Object fieldValue = fieldAccessor.getObject(originObj); - fieldAccessor.putObject( - newObj, copyFieldValue(copyContext, fieldValue, fieldInfo.dispatchId)); - continue; - } if (fieldInfo.isPrimitiveField) { - copySetPrimitiveField(originObj, newObj, fieldOffset, fieldInfo.dispatchId); + copySetPrimitiveField(originObj, newObj, fieldAccessor, fieldInfo.dispatchId); } else { - copySetNotPrimitiveField(copyContext, originObj, newObj, fieldOffset, fieldInfo.dispatchId); + copySetNotPrimitiveField( + copyContext, originObj, newObj, fieldAccessor, fieldInfo.dispatchId); } } } @@ -1129,33 +964,33 @@ private static Object copyFieldValue(CopyContext copyContext, Object fieldValue, } private static void copySetPrimitiveField( - Object originObj, Object newObj, long fieldOffset, int typeId) { + Object originObj, Object newObj, FieldAccessor fieldAccessor, int typeId) { switch (typeId) { case DispatchId.BOOL: - UnsafeOps.putBoolean(newObj, fieldOffset, UnsafeOps.getBoolean(originObj, fieldOffset)); + fieldAccessor.putBoolean(newObj, fieldAccessor.getBoolean(originObj)); break; case DispatchId.INT8: - UnsafeOps.putByte(newObj, fieldOffset, UnsafeOps.getByte(originObj, fieldOffset)); + fieldAccessor.putByte(newObj, fieldAccessor.getByte(originObj)); break; case DispatchId.UINT8: - UnsafeOps.putInt(newObj, fieldOffset, UnsafeOps.getInt(originObj, fieldOffset)); + fieldAccessor.putInt(newObj, fieldAccessor.getInt(originObj)); break; case DispatchId.CHAR: - UnsafeOps.putChar(newObj, fieldOffset, UnsafeOps.getChar(originObj, fieldOffset)); + fieldAccessor.putChar(newObj, fieldAccessor.getChar(originObj)); break; case DispatchId.INT16: - UnsafeOps.putShort(newObj, fieldOffset, UnsafeOps.getShort(originObj, fieldOffset)); + fieldAccessor.putShort(newObj, fieldAccessor.getShort(originObj)); break; case DispatchId.UINT16: - UnsafeOps.putInt(newObj, fieldOffset, UnsafeOps.getInt(originObj, fieldOffset)); + fieldAccessor.putInt(newObj, fieldAccessor.getInt(originObj)); break; case DispatchId.INT32: case DispatchId.VARINT32: - UnsafeOps.putInt(newObj, fieldOffset, UnsafeOps.getInt(originObj, fieldOffset)); + fieldAccessor.putInt(newObj, fieldAccessor.getInt(originObj)); break; case DispatchId.UINT32: case DispatchId.VAR_UINT32: - UnsafeOps.putLong(newObj, fieldOffset, UnsafeOps.getLong(originObj, fieldOffset)); + fieldAccessor.putLong(newObj, fieldAccessor.getLong(originObj)); break; case DispatchId.INT64: case DispatchId.VARINT64: @@ -1163,13 +998,13 @@ private static void copySetPrimitiveField( case DispatchId.UINT64: case DispatchId.VAR_UINT64: case DispatchId.TAGGED_UINT64: - UnsafeOps.putLong(newObj, fieldOffset, UnsafeOps.getLong(originObj, fieldOffset)); + fieldAccessor.putLong(newObj, fieldAccessor.getLong(originObj)); break; case DispatchId.FLOAT32: - UnsafeOps.putFloat(newObj, fieldOffset, UnsafeOps.getFloat(originObj, fieldOffset)); + fieldAccessor.putFloat(newObj, fieldAccessor.getFloat(originObj)); break; case DispatchId.FLOAT64: - UnsafeOps.putDouble(newObj, fieldOffset, UnsafeOps.getDouble(originObj, fieldOffset)); + fieldAccessor.putDouble(newObj, fieldAccessor.getDouble(originObj)); break; default: throw new RuntimeException("Unknown primitive type: " + typeId); @@ -1177,109 +1012,54 @@ private static void copySetPrimitiveField( } private static void copySetNotPrimitiveField( - CopyContext copyContext, Object originObj, Object newObj, long fieldOffset, int typeId) { - switch (typeId) { - case DispatchId.BOOL: - case DispatchId.INT8: - case DispatchId.UINT8: - case DispatchId.EXT_UINT8: - case DispatchId.CHAR: - case DispatchId.INT16: - case DispatchId.UINT16: - case DispatchId.EXT_UINT16: - case DispatchId.INT32: - case DispatchId.VARINT32: - case DispatchId.UINT32: - case DispatchId.EXT_UINT32: - case DispatchId.VAR_UINT32: - case DispatchId.EXT_VAR_UINT32: - case DispatchId.INT64: - case DispatchId.VARINT64: - case DispatchId.TAGGED_INT64: - case DispatchId.UINT64: - case DispatchId.EXT_UINT64: - case DispatchId.VAR_UINT64: - case DispatchId.EXT_VAR_UINT64: - case DispatchId.TAGGED_UINT64: - case DispatchId.FLOAT32: - case DispatchId.FLOAT64: - case DispatchId.FLOAT16: - case DispatchId.BFLOAT16: - case DispatchId.STRING: - UnsafeOps.putObject(newObj, fieldOffset, UnsafeOps.getObject(originObj, fieldOffset)); - break; - default: - UnsafeOps.putObject( - newObj, - fieldOffset, - copyContext.copyObject(UnsafeOps.getObject(originObj, fieldOffset))); - } + CopyContext copyContext, + Object originObj, + Object newObj, + FieldAccessor fieldAccessor, + int typeId) { + Object fieldValue = fieldAccessor.getObject(originObj); + fieldAccessor.putObject(newObj, copyFieldValue(copyContext, fieldValue, typeId)); } - private Object copyPrimitiveField(Object targetObject, long fieldOffset, int typeId) { + private Object copyPrimitiveField(Object targetObject, FieldAccessor fieldAccessor, int typeId) { switch (typeId) { case DispatchId.BOOL: - return UnsafeOps.getBoolean(targetObject, fieldOffset); + return fieldAccessor.getBoolean(targetObject); case DispatchId.INT8: - return UnsafeOps.getByte(targetObject, fieldOffset); + return fieldAccessor.getByte(targetObject); case DispatchId.UINT8: - return UnsafeOps.getInt(targetObject, fieldOffset); + return fieldAccessor.getInt(targetObject); case DispatchId.CHAR: - return UnsafeOps.getChar(targetObject, fieldOffset); + return fieldAccessor.getChar(targetObject); case DispatchId.INT16: - return UnsafeOps.getShort(targetObject, fieldOffset); + return fieldAccessor.getShort(targetObject); case DispatchId.UINT16: - return UnsafeOps.getInt(targetObject, fieldOffset); + return fieldAccessor.getInt(targetObject); case DispatchId.INT32: case DispatchId.VARINT32: - return UnsafeOps.getInt(targetObject, fieldOffset); + return fieldAccessor.getInt(targetObject); case DispatchId.UINT32: case DispatchId.VAR_UINT32: - return UnsafeOps.getLong(targetObject, fieldOffset); + return fieldAccessor.getLong(targetObject); case DispatchId.FLOAT32: - return UnsafeOps.getFloat(targetObject, fieldOffset); + return fieldAccessor.getFloat(targetObject); case DispatchId.INT64: case DispatchId.VARINT64: case DispatchId.TAGGED_INT64: case DispatchId.UINT64: case DispatchId.VAR_UINT64: case DispatchId.TAGGED_UINT64: - return UnsafeOps.getLong(targetObject, fieldOffset); + return fieldAccessor.getLong(targetObject); case DispatchId.FLOAT64: - return UnsafeOps.getDouble(targetObject, fieldOffset); + return fieldAccessor.getDouble(targetObject); default: throw new RuntimeException("Unknown primitive type: " + typeId); } } private Object copyNotPrimitiveField( - CopyContext copyContext, Object targetObject, long fieldOffset, int typeId) { - switch (typeId) { - case DispatchId.BOOL: - case DispatchId.INT8: - case DispatchId.UINT8: - case DispatchId.CHAR: - case DispatchId.INT16: - case DispatchId.UINT16: - case DispatchId.INT32: - case DispatchId.VARINT32: - case DispatchId.UINT32: - case DispatchId.VAR_UINT32: - case DispatchId.FLOAT32: - case DispatchId.INT64: - case DispatchId.VARINT64: - case DispatchId.TAGGED_INT64: - case DispatchId.UINT64: - case DispatchId.VAR_UINT64: - case DispatchId.TAGGED_UINT64: - case DispatchId.FLOAT64: - case DispatchId.FLOAT16: - case DispatchId.BFLOAT16: - case DispatchId.STRING: - return UnsafeOps.getObject(targetObject, fieldOffset); - default: - return copyContext.copyObject(UnsafeOps.getObject(targetObject, fieldOffset)); - } + CopyContext copyContext, Object targetObject, FieldAccessor fieldAccessor, int typeId) { + return copyFieldValue(copyContext, fieldAccessor.getObject(targetObject), typeId); } private SerializationFieldInfo[] buildFieldsInfo() { diff --git a/java/fory-core/src/main/java/org/apache/fory/util/DefaultValueUtils.java b/java/fory-core/src/main/java/org/apache/fory/util/DefaultValueUtils.java index 554a8d7f89..a6713e8717 100644 --- a/java/fory-core/src/main/java/org/apache/fory/util/DefaultValueUtils.java +++ b/java/fory-core/src/main/java/org/apache/fory/util/DefaultValueUtils.java @@ -34,7 +34,6 @@ import org.apache.fory.logging.Logger; import org.apache.fory.logging.LoggerFactory; import org.apache.fory.platform.AndroidSupport; -import org.apache.fory.platform.UnsafeOps; import org.apache.fory.reflect.FieldAccessor; import org.apache.fory.resolver.TypeResolver; import org.apache.fory.type.ScalaTypes; @@ -418,35 +417,30 @@ public static void setDefaultValues(Object obj, DefaultValueField[] defaultValue FieldAccessor fieldAccessor = defaultField.getFieldAccessor(); if (fieldAccessor != null) { Object defaultValue = defaultField.getDefaultValue(); - if (AndroidSupport.IS_ANDROID) { - fieldAccessor.set(obj, defaultValue); - continue; - } - long fieldOffset = fieldAccessor.getFieldOffset(); switch (defaultField.dispatchId) { case Types.BOOL: - UnsafeOps.putBoolean(obj, fieldOffset, (Boolean) defaultValue); + fieldAccessor.putBoolean(obj, (Boolean) defaultValue); break; case Types.INT8: - UnsafeOps.putByte(obj, fieldOffset, (Byte) defaultValue); + fieldAccessor.putByte(obj, (Byte) defaultValue); break; case Types.INT16: - UnsafeOps.putShort(obj, fieldOffset, (Short) defaultValue); + fieldAccessor.putShort(obj, (Short) defaultValue); break; case Types.INT32: case Types.VARINT32: - UnsafeOps.putInt(obj, fieldOffset, (Integer) defaultValue); + fieldAccessor.putInt(obj, (Integer) defaultValue); break; case Types.INT64: case Types.VARINT64: case Types.TAGGED_INT64: - UnsafeOps.putLong(obj, fieldOffset, (Long) defaultValue); + fieldAccessor.putLong(obj, (Long) defaultValue); break; case Types.FLOAT32: - UnsafeOps.putFloat(obj, fieldOffset, (Float) defaultValue); + fieldAccessor.putFloat(obj, (Float) defaultValue); break; case Types.FLOAT64: - UnsafeOps.putDouble(obj, fieldOffset, (Double) defaultValue); + fieldAccessor.putDouble(obj, (Double) defaultValue); break; default: // Object type (including String, char, boxed types not covered above) diff --git a/java/fory-core/src/test/java/org/apache/fory/reflect/FieldAccessorTest.java b/java/fory-core/src/test/java/org/apache/fory/reflect/FieldAccessorTest.java index 32d1d207ad..68209013fb 100644 --- a/java/fory-core/src/test/java/org/apache/fory/reflect/FieldAccessorTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/reflect/FieldAccessorTest.java @@ -46,14 +46,23 @@ public void testGeneratedAccessor() throws Exception { Assert.assertEquals(f1.get(struct), 10); f1.set(struct, 20); Assert.assertEquals(f1.get(struct), 20); + Assert.assertEquals(f1.getInt(struct), 20); + f1.putInt(struct, 30); + Assert.assertEquals(f1.getInt(struct), 30); GeneratedAccessor f2 = new GeneratedAccessor(TestStruct.class.getDeclaredField("f2")); Assert.assertEquals(f2.get(struct), true); f2.set(struct, false); Assert.assertEquals(f2.get(struct), false); + Assert.assertFalse(f2.getBoolean(struct)); + f2.putBoolean(struct, true); + Assert.assertTrue(f2.getBoolean(struct)); GeneratedAccessor f3 = new GeneratedAccessor(TestStruct.class.getDeclaredField("f3")); Assert.assertEquals(f3.get(struct), "str"); f3.set(struct, "a"); Assert.assertEquals(f3.get(struct), "a"); + Assert.assertEquals(f3.getObject(struct), "a"); + f3.putObject(struct, "b"); + Assert.assertEquals(f3.getObject(struct), "b"); } @Test @@ -107,7 +116,6 @@ private static void assertAccessor( FieldAccessor accessor = FieldAccessor.createAccessor(field); check( accessor instanceof ReflectionFieldAccessor, "Expected reflection accessor for " + field); - check(accessor.getFieldOffset() == -1, "Android field accessor should not expose offsets"); checkEquals(accessor.get(fields), expected, "initial " + fieldName); accessor.set(fields, replacement); checkEquals(accessor.get(fields), replacement, "updated " + fieldName); From 4179012fd2b4455529097dad6887db821d572589 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sat, 23 May 2026 21:22:21 +0800 Subject: [PATCH 05/69] remove unused code --- .../org/apache/fory/memory/LittleEndian.java | 33 ------------------- 1 file changed, 33 deletions(-) diff --git a/java/fory-core/src/main/java/org/apache/fory/memory/LittleEndian.java b/java/fory-core/src/main/java/org/apache/fory/memory/LittleEndian.java index fa0a831eff..0e0c2d246a 100644 --- a/java/fory-core/src/main/java/org/apache/fory/memory/LittleEndian.java +++ b/java/fory-core/src/main/java/org/apache/fory/memory/LittleEndian.java @@ -58,23 +58,6 @@ private static int bigWriteUint36(byte[] arr, int index, long v) { return 5; } - public static void putInt32(Object o, long pos, int value) { - if (!NativeByteOrder.IS_LITTLE_ENDIAN) { - value = Integer.reverseBytes(value); - } - UnsafeOps.putInt(o, pos, value); - } - - public static int getInt32(Object o, long pos) { - int i = UnsafeOps.getInt(o, pos); - return NativeByteOrder.IS_LITTLE_ENDIAN ? i : Integer.reverseBytes(i); - } - - public static long getInt64(Object o, long pos) { - long v = UnsafeOps.getLong(o, pos); - return NativeByteOrder.IS_LITTLE_ENDIAN ? v : Long.reverseBytes(v); - } - public static long getInt64(byte[] o, int index) { if (AndroidSupport.IS_ANDROID) { return MemoryOps.getInt64(o, index); @@ -93,20 +76,4 @@ public static void putInt64(byte[] o, int index, long value) { } UnsafeOps.putLong(o, UnsafeOps.BYTE_ARRAY_OFFSET + index, value); } - - public static void putFloat32(Object o, long pos, float value) { - int v = Float.floatToRawIntBits(value); - if (!NativeByteOrder.IS_LITTLE_ENDIAN) { - v = Integer.reverseBytes(v); - } - UnsafeOps.putInt(o, pos, v); - } - - public static void putFloat64(Object o, long pos, double value) { - long v = Double.doubleToRawLongBits(value); - if (!NativeByteOrder.IS_LITTLE_ENDIAN) { - v = Long.reverseBytes(v); - } - UnsafeOps.putLong(o, pos, v); - } } From c3f76efc70ce9e5088a783530d59954fda30d7b1 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sat, 23 May 2026 14:10:16 +0800 Subject: [PATCH 06/69] remove UNSAFE.throwException usage --- .../apache/fory/util/unsafe/_JDKAccess.java | 22 +++++++------------ 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/java/fory-core/src/main/java/org/apache/fory/util/unsafe/_JDKAccess.java b/java/fory-core/src/main/java/org/apache/fory/util/unsafe/_JDKAccess.java index e9e8774f60..d41b302679 100644 --- a/java/fory-core/src/main/java/org/apache/fory/util/unsafe/_JDKAccess.java +++ b/java/fory-core/src/main/java/org/apache/fory/util/unsafe/_JDKAccess.java @@ -132,8 +132,7 @@ public static Function makeJDKFunction( boxedMethodType(handle.type())); return (Function) callSite.getTarget().invokeExact(); } catch (Throwable e) { - UNSAFE.throwException(e); - throw new IllegalStateException(e); + throw ExceptionUtils.throwException(e); } } @@ -153,8 +152,7 @@ public static Consumer makeJDKConsumer(Lookup lookup, MethodHandle handle boxedMethodType(handle.type())); return (Consumer) callSite.getTarget().invokeExact(); } catch (Throwable e) { - UNSAFE.throwException(e); - throw new IllegalStateException(e); + throw ExceptionUtils.throwException(e); } } @@ -174,8 +172,7 @@ public static BiConsumer makeJDKBiConsumer(Lookup lookup, MethodHan boxedMethodType(handle.type())); return (BiConsumer) callSite.getTarget().invokeExact(); } catch (Throwable e) { - UNSAFE.throwException(e); - throw new IllegalStateException(e); + throw ExceptionUtils.throwException(e); } } @@ -208,8 +205,7 @@ public static T makeFunction(Lookup lookup, MethodHandle handle, Method meth instantiatedMethodType); return (T) callSite.getTarget().invokeExact(); } catch (Throwable e) { - UNSAFE.throwException(e); - throw new IllegalStateException(e); + throw ExceptionUtils.throwException(e); } } @@ -243,8 +239,7 @@ public static T makeFunction(Lookup lookup, MethodHandle handle, Class fu // FIXME(chaokunyang) why use invokeExact will fail. return (T) callSite.getTarget().invoke(); } catch (Throwable e) { - UNSAFE.throwException(e); - throw new IllegalStateException(e); + throw ExceptionUtils.throwException(e); } } @@ -296,8 +291,7 @@ public static Object makeGetterFunction( // represented by handle, then exception will be thrown. return makeGetterFunction(lookup, handle, Object.class); } catch (Throwable e) { - UNSAFE.throwException(e); - throw new IllegalStateException(e); + throw ExceptionUtils.throwException(e); } } @@ -315,7 +309,7 @@ public static Object getModule(Class cls) { try { return getModuleMethod.invoke(cls); } catch (IllegalAccessException | InvocationTargetException e) { - throw new RuntimeException(e); + throw ExceptionUtils.throwException(e); } } @@ -332,7 +326,7 @@ public static Object addReads(Object thisModule, Object otherModule) { } return addReadsHandle.invoke(thisModule, otherModule); } catch (Throwable e) { - throw new RuntimeException(e); + throw ExceptionUtils.throwException(e); } } } From 0add063f2f505d5d08ddb1d815852257bfbace95 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sat, 23 May 2026 22:11:32 +0800 Subject: [PATCH 07/69] refactor(java): add Fory byte array streams --- .../src/main/java/org/apache/fory/Fory.java | 10 +-- .../fory/io/ForyByteArrayInputStream.java | 82 +++++++++++++++++++ .../fory/io/ForyByteArrayOutputStream.java | 63 ++++++++++++++ .../org/apache/fory/memory/MemoryUtils.java | 75 ++++------------- .../test/java/org/apache/fory/StreamTest.java | 6 ++ .../apache/fory/memory/MemoryBufferTest.java | 23 ++++++ .../serializer/AndroidDynamicFeatureTest.java | 45 +++++----- 7 files changed, 217 insertions(+), 87 deletions(-) create mode 100644 java/fory-core/src/main/java/org/apache/fory/io/ForyByteArrayInputStream.java create mode 100644 java/fory-core/src/main/java/org/apache/fory/io/ForyByteArrayOutputStream.java diff --git a/java/fory-core/src/main/java/org/apache/fory/Fory.java b/java/fory-core/src/main/java/org/apache/fory/Fory.java index a4bf27ae37..533f0ebb57 100644 --- a/java/fory-core/src/main/java/org/apache/fory/Fory.java +++ b/java/fory-core/src/main/java/org/apache/fory/Fory.java @@ -19,7 +19,6 @@ package org.apache.fory; -import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.OutputStream; import java.nio.ByteBuffer; @@ -46,13 +45,13 @@ import org.apache.fory.exception.DeserializationException; import org.apache.fory.exception.ForyException; import org.apache.fory.exception.SerializationException; +import org.apache.fory.io.ForyByteArrayOutputStream; import org.apache.fory.io.ForyInputStream; import org.apache.fory.io.ForyReadableChannel; import org.apache.fory.logging.Logger; import org.apache.fory.logging.LoggerFactory; import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.memory.MemoryUtils; -import org.apache.fory.platform.AndroidSupport; import org.apache.fory.resolver.ClassResolver; import org.apache.fory.resolver.SharedRegistry; import org.apache.fory.resolver.TypeChecker; @@ -564,12 +563,13 @@ public T copy(T obj) { private void serializeToStream(OutputStream outputStream, Consumer function) { MemoryBuffer buf = getBuffer(); - if (!AndroidSupport.IS_ANDROID && outputStream.getClass() == ByteArrayOutputStream.class) { + if (outputStream instanceof ForyByteArrayOutputStream) { + ForyByteArrayOutputStream byteArrayStream = (ForyByteArrayOutputStream) outputStream; byte[] oldBytes = buf.getHeapMemory(); // Note: This should not be null. assert oldBytes != null; - MemoryUtils.wrap((ByteArrayOutputStream) outputStream, buf); + MemoryUtils.wrap(byteArrayStream, buf); function.accept(buf); - MemoryUtils.wrap(buf, (ByteArrayOutputStream) outputStream); + MemoryUtils.wrap(buf, byteArrayStream); buf.pointTo(oldBytes, 0, oldBytes.length); resetBuffer(); } else { diff --git a/java/fory-core/src/main/java/org/apache/fory/io/ForyByteArrayInputStream.java b/java/fory-core/src/main/java/org/apache/fory/io/ForyByteArrayInputStream.java new file mode 100644 index 0000000000..a622b78d36 --- /dev/null +++ b/java/fory-core/src/main/java/org/apache/fory/io/ForyByteArrayInputStream.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.io; + +import java.io.ByteArrayInputStream; +import org.apache.fory.util.Preconditions; + +/** A {@link ByteArrayInputStream} with public accessors for its protected state. */ +public class ForyByteArrayInputStream extends ByteArrayInputStream { + public ForyByteArrayInputStream(byte[] buffer) { + super(buffer); + } + + public ForyByteArrayInputStream(byte[] buffer, int offset, int length) { + super(buffer, offset, length); + } + + public byte[] getBuffer() { + return buf; + } + + public void setBuffer(byte[] buffer) { + buf = Preconditions.checkNotNull(buffer); + pos = 0; + mark = 0; + count = buffer.length; + } + + public void setBuffer(byte[] buffer, int offset, int length) { + Preconditions.checkNotNull(buffer); + Preconditions.checkArgument(offset >= 0 && length >= 0 && length <= buffer.length - offset); + buf = buffer; + pos = offset; + mark = offset; + count = offset + length; + } + + public int getPosition() { + return pos; + } + + public void setPosition(int position) { + Preconditions.checkArgument(position >= 0 && position <= count); + pos = position; + } + + public int getMark() { + return mark; + } + + public void setMark(int mark) { + Preconditions.checkArgument(mark >= 0 && mark <= count); + this.mark = mark; + } + + public int getCount() { + return count; + } + + public void setCount(int count) { + Preconditions.checkArgument(count >= 0 && count <= buf.length); + Preconditions.checkArgument(pos <= count && mark <= count); + this.count = count; + } +} diff --git a/java/fory-core/src/main/java/org/apache/fory/io/ForyByteArrayOutputStream.java b/java/fory-core/src/main/java/org/apache/fory/io/ForyByteArrayOutputStream.java new file mode 100644 index 0000000000..572febfd78 --- /dev/null +++ b/java/fory-core/src/main/java/org/apache/fory/io/ForyByteArrayOutputStream.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.io; + +import java.io.ByteArrayOutputStream; +import org.apache.fory.util.Preconditions; + +/** A {@link ByteArrayOutputStream} with public accessors for its backing byte array and count. */ +public class ForyByteArrayOutputStream extends ByteArrayOutputStream { + public ForyByteArrayOutputStream() { + super(); + } + + public ForyByteArrayOutputStream(int size) { + super(size); + } + + public ForyByteArrayOutputStream(byte[] buffer) { + setBuffer(buffer); + } + + public ForyByteArrayOutputStream(byte[] buffer, int count) { + setBuffer(buffer); + setCount(count); + } + + public byte[] getBuffer() { + return buf; + } + + public void setBuffer(byte[] buffer) { + buf = Preconditions.checkNotNull(buffer); + if (count > buffer.length) { + count = buffer.length; + } + } + + public int getCount() { + return count; + } + + public void setCount(int count) { + Preconditions.checkArgument(count >= 0 && count <= buf.length); + this.count = count; + } +} diff --git a/java/fory-core/src/main/java/org/apache/fory/memory/MemoryUtils.java b/java/fory-core/src/main/java/org/apache/fory/memory/MemoryUtils.java index bb23ca4994..6eef8aa81c 100644 --- a/java/fory-core/src/main/java/org/apache/fory/memory/MemoryUtils.java +++ b/java/fory-core/src/main/java/org/apache/fory/memory/MemoryUtils.java @@ -19,11 +19,10 @@ package org.apache.fory.memory; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; import java.nio.ByteBuffer; +import org.apache.fory.io.ForyByteArrayInputStream; +import org.apache.fory.io.ForyByteArrayOutputStream; import org.apache.fory.platform.AndroidSupport; -import org.apache.fory.platform.UnsafeOps; import org.apache.fory.util.Preconditions; /** Memory utils for fory. */ @@ -71,77 +70,39 @@ public static MemoryBuffer wrap(ByteBuffer buffer) { } } - // Lazy load offset and also follow graalvm offset auto replace pattern. - private static class Offset { - private static final long BAS_BUF_BUF; - private static final long BAS_BUF_COUNT; - private static final long BIS_BUF_BUF; - private static final long BIS_BUF_POS; - private static final long BIS_BUF_COUNT; - - static { - try { - BAS_BUF_BUF = - UnsafeOps.objectFieldOffset(ByteArrayOutputStream.class.getDeclaredField("buf")); - BAS_BUF_COUNT = - UnsafeOps.objectFieldOffset(ByteArrayOutputStream.class.getDeclaredField("count")); - BIS_BUF_BUF = - UnsafeOps.objectFieldOffset(ByteArrayInputStream.class.getDeclaredField("buf")); - BIS_BUF_POS = - UnsafeOps.objectFieldOffset(ByteArrayInputStream.class.getDeclaredField("pos")); - BIS_BUF_COUNT = - UnsafeOps.objectFieldOffset(ByteArrayInputStream.class.getDeclaredField("count")); - } catch (NoSuchFieldException e) { - throw new RuntimeException(e); - } - } - } - /** - * Wrap a {@link ByteArrayOutputStream} into a {@link MemoryBuffer}. The writerIndex of buffer - * will be the count of stream. + * Wrap a {@link ForyByteArrayOutputStream} into a {@link MemoryBuffer}. The writerIndex of buffer + * will be the count of the stream. */ - public static void wrap(ByteArrayOutputStream stream, MemoryBuffer buffer) { - if (AndroidSupport.IS_ANDROID) { - throw new UnsupportedOperationException( - "ByteArrayOutputStream direct wrapping is not supported on Android"); - } + public static void wrap(ForyByteArrayOutputStream stream, MemoryBuffer buffer) { Preconditions.checkNotNull(stream); - byte[] buf = (byte[]) UnsafeOps.getObject(stream, Offset.BAS_BUF_BUF); - int count = UnsafeOps.getInt(stream, Offset.BAS_BUF_COUNT); + byte[] buf = stream.getBuffer(); + int count = stream.getCount(); buffer.pointTo(buf, 0, buf.length); buffer.writerIndex(count); } /** - * Wrap a @link MemoryBuffer} into a {@link ByteArrayOutputStream}. The count of stream will be - * the writerIndex of buffer. + * Wrap a {@link MemoryBuffer} into a {@link ForyByteArrayOutputStream}. The count of the stream + * will be the writerIndex of buffer. */ - public static void wrap(MemoryBuffer buffer, ByteArrayOutputStream stream) { - if (AndroidSupport.IS_ANDROID) { - throw new UnsupportedOperationException( - "ByteArrayOutputStream direct wrapping is not supported on Android"); - } + public static void wrap(MemoryBuffer buffer, ForyByteArrayOutputStream stream) { Preconditions.checkNotNull(stream); byte[] bytes = buffer.getHeapMemory(); Preconditions.checkNotNull(bytes); - UnsafeOps.putObject(stream, Offset.BAS_BUF_BUF, bytes); - UnsafeOps.putInt(stream, Offset.BAS_BUF_COUNT, buffer.writerIndex()); + stream.setBuffer(bytes); + stream.setCount(buffer.writerIndex()); } /** - * Wrap a {@link ByteArrayInputStream} into a {@link MemoryBuffer}. The readerIndex of buffer will - * be the pos of stream. + * Wrap a {@link ForyByteArrayInputStream} into a {@link MemoryBuffer}. The readerIndex of buffer + * will be the position of the stream. */ - public static void wrap(ByteArrayInputStream stream, MemoryBuffer buffer) { - if (AndroidSupport.IS_ANDROID) { - throw new UnsupportedOperationException( - "ByteArrayInputStream direct wrapping is not supported on Android"); - } + public static void wrap(ForyByteArrayInputStream stream, MemoryBuffer buffer) { Preconditions.checkNotNull(stream); - byte[] buf = (byte[]) UnsafeOps.getObject(stream, Offset.BIS_BUF_BUF); - int count = UnsafeOps.getInt(stream, Offset.BIS_BUF_COUNT); - int pos = UnsafeOps.getInt(stream, Offset.BIS_BUF_POS); + byte[] buf = stream.getBuffer(); + int count = stream.getCount(); + int pos = stream.getPosition(); buffer.pointTo(buf, 0, count); buffer.readerIndex(pos); } diff --git a/java/fory-core/src/test/java/org/apache/fory/StreamTest.java b/java/fory-core/src/test/java/org/apache/fory/StreamTest.java index e25b486493..40860d8b72 100644 --- a/java/fory-core/src/test/java/org/apache/fory/StreamTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/StreamTest.java @@ -38,6 +38,7 @@ import java.util.List; import java.util.Map; import org.apache.fory.exception.DeserializationException; +import org.apache.fory.io.ForyByteArrayOutputStream; import org.apache.fory.io.ForyInputStream; import org.apache.fory.io.ForyReadableChannel; import org.apache.fory.io.ForyStreamReader; @@ -133,6 +134,11 @@ public void testBufferReset() { assertEquals(o, new byte[1000 * 1000]); assertEquals(fory.deserialize(bas.toByteArray()), new byte[1000 * 1000]); + ForyByteArrayOutputStream foryBas = new ForyByteArrayOutputStream(); + fory.serialize(foryBas, "fory-stream"); + checkBuffer(fory); + assertEquals(fory.deserialize(foryBas.toByteArray()), "fory-stream"); + bas.reset(); fory.serialize(bas, new byte[1000 * 1000]); checkBuffer(fory); diff --git a/java/fory-core/src/test/java/org/apache/fory/memory/MemoryBufferTest.java b/java/fory-core/src/test/java/org/apache/fory/memory/MemoryBufferTest.java index 1af57bf458..5d035e30cb 100644 --- a/java/fory-core/src/test/java/org/apache/fory/memory/MemoryBufferTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/memory/MemoryBufferTest.java @@ -30,6 +30,8 @@ import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.util.Random; +import org.apache.fory.io.ForyByteArrayInputStream; +import org.apache.fory.io.ForyByteArrayOutputStream; import org.apache.fory.platform.AndroidSupport; import org.testng.Assert; import org.testng.annotations.Test; @@ -81,6 +83,27 @@ public void testBufferWrite() { assertEquals(buffer.readerIndex(), buffer.writerIndex()); } + @Test + public void testForyByteArrayStreamWrap() { + ForyByteArrayOutputStream outputStream = new ForyByteArrayOutputStream(8); + outputStream.write(new byte[] {1, 2, 3}, 0, 3); + MemoryBuffer buffer = MemoryUtils.buffer(1); + MemoryUtils.wrap(outputStream, buffer); + assertEquals(buffer.getHeapMemory(), outputStream.getBuffer()); + assertEquals(buffer.writerIndex(), 3); + buffer.writeByte((byte) 4); + MemoryUtils.wrap(buffer, outputStream); + assertEquals(outputStream.getCount(), 4); + assertEquals(outputStream.getBuffer()[3], (byte) 4); + + ForyByteArrayInputStream inputStream = new ForyByteArrayInputStream(new byte[] {5, 6, 7}); + assertEquals(inputStream.read(), 5); + MemoryUtils.wrap(inputStream, buffer); + assertEquals(buffer.getHeapMemory(), inputStream.getBuffer()); + assertEquals(buffer.readerIndex(), 1); + assertEquals(buffer.readByte(), (byte) 6); + } + @Test public void testAndroidHeapMemoryBufferPaths() throws Exception { String javaBin = diff --git a/java/fory-core/src/test/java/org/apache/fory/serializer/AndroidDynamicFeatureTest.java b/java/fory-core/src/test/java/org/apache/fory/serializer/AndroidDynamicFeatureTest.java index 6d08427b52..3be0ea6756 100644 --- a/java/fory-core/src/test/java/org/apache/fory/serializer/AndroidDynamicFeatureTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/serializer/AndroidDynamicFeatureTest.java @@ -19,7 +19,6 @@ package org.apache.fory.serializer; -import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; @@ -39,6 +38,9 @@ import org.apache.fory.Fory; import org.apache.fory.context.ReadContext; import org.apache.fory.context.WriteContext; +import org.apache.fory.io.ForyByteArrayInputStream; +import org.apache.fory.io.ForyByteArrayOutputStream; +import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.memory.MemoryUtils; import org.apache.fory.platform.AndroidSupport; import org.apache.fory.resolver.TypeResolver; @@ -87,7 +89,7 @@ public static void main(String[] args) { LambdaSerializer.STUB_LAMBDA_CLASS == LambdaSerializer.ReplaceStub.class, "Android must not create a runtime lambda stub class"); verifyReflectiveGetter(); - verifyMemoryUtilsStreamWrapGuards(); + verifyMemoryUtilsStreamWraps(); verifyXlangUnion(); verifyFory(false); @@ -172,18 +174,22 @@ private static void verifyOutputStreamSerialization(Fory fory) { checkEquals(fory.deserialize(outputStream.toByteArray()), value, "OutputStream round trip"); } - private static void verifyMemoryUtilsStreamWrapGuards() { - expectUnsupportedAndroidWrap( - () -> MemoryUtils.wrap(new ByteArrayOutputStream(), MemoryUtils.buffer(8)), - "ByteArrayOutputStream direct wrapping"); - expectUnsupportedAndroidWrap( - () -> MemoryUtils.wrap(MemoryUtils.buffer(8), new ByteArrayOutputStream()), - "ByteArrayOutputStream direct wrapping"); - expectUnsupportedAndroidWrap( - () -> - MemoryUtils.wrap( - new ByteArrayInputStream(new byte[] {1, 2, 3}), MemoryUtils.buffer(8)), - "ByteArrayInputStream direct wrapping"); + private static void verifyMemoryUtilsStreamWraps() { + MemoryBuffer buffer = MemoryUtils.buffer(8); + ForyByteArrayOutputStream outputStream = new ForyByteArrayOutputStream(8); + outputStream.write(new byte[] {1, 2}, 0, 2); + MemoryUtils.wrap(outputStream, buffer); + checkEquals(buffer.writerIndex(), 2, "Output stream writer index"); + buffer.writeByte((byte) 3); + MemoryUtils.wrap(buffer, outputStream); + checkEquals(outputStream.getCount(), 3, "Output stream count"); + checkEquals(outputStream.getBuffer()[2], (byte) 3, "Output stream buffer"); + + ForyByteArrayInputStream inputStream = new ForyByteArrayInputStream(new byte[] {4, 5, 6}); + checkEquals(inputStream.read(), 4, "Input stream first byte"); + MemoryUtils.wrap(inputStream, buffer); + checkEquals(buffer.readerIndex(), 1, "Input stream reader index"); + checkEquals(buffer.readByte(), (byte) 5, "Input stream buffer"); } private static void verifyXlangUnion() { @@ -257,17 +263,6 @@ private static void expectUnsupported(Serializer serializer) { } } - private static void expectUnsupportedAndroidWrap(Runnable operation, String messageFragment) { - try { - operation.run(); - throw new AssertionError("Expected Android unsafe stream wrapping to fail"); - } catch (UnsupportedOperationException expected) { - check( - expected.getMessage().contains(messageFragment), - "Unexpected unsupported message: " + expected.getMessage()); - } - } - private static void check(boolean value, String message) { if (!value) { throw new AssertionError(message); From 4603f536f420032fec6f9903614ed8f32e987b74 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sat, 23 May 2026 22:28:48 +0800 Subject: [PATCH 08/69] refactor(java): route byte array stream wrapping through JDK access --- .../src/main/java/org/apache/fory/Fory.java | 10 +-- .../fory/io/ForyByteArrayInputStream.java | 82 ------------------- .../fory/io/ForyByteArrayOutputStream.java | 63 -------------- .../org/apache/fory/memory/MemoryUtils.java | 55 ++++++------- .../apache/fory/util/unsafe/_JDKAccess.java | 54 ++++++++++++ .../test/java/org/apache/fory/StreamTest.java | 6 -- .../apache/fory/memory/MemoryBufferTest.java | 19 +++-- .../serializer/AndroidDynamicFeatureTest.java | 48 ++++++----- 8 files changed, 124 insertions(+), 213 deletions(-) delete mode 100644 java/fory-core/src/main/java/org/apache/fory/io/ForyByteArrayInputStream.java delete mode 100644 java/fory-core/src/main/java/org/apache/fory/io/ForyByteArrayOutputStream.java diff --git a/java/fory-core/src/main/java/org/apache/fory/Fory.java b/java/fory-core/src/main/java/org/apache/fory/Fory.java index 533f0ebb57..140d83f28c 100644 --- a/java/fory-core/src/main/java/org/apache/fory/Fory.java +++ b/java/fory-core/src/main/java/org/apache/fory/Fory.java @@ -19,6 +19,7 @@ package org.apache.fory; +import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.OutputStream; import java.nio.ByteBuffer; @@ -45,7 +46,6 @@ import org.apache.fory.exception.DeserializationException; import org.apache.fory.exception.ForyException; import org.apache.fory.exception.SerializationException; -import org.apache.fory.io.ForyByteArrayOutputStream; import org.apache.fory.io.ForyInputStream; import org.apache.fory.io.ForyReadableChannel; import org.apache.fory.logging.Logger; @@ -563,13 +563,13 @@ public T copy(T obj) { private void serializeToStream(OutputStream outputStream, Consumer function) { MemoryBuffer buf = getBuffer(); - if (outputStream instanceof ForyByteArrayOutputStream) { - ForyByteArrayOutputStream byteArrayStream = (ForyByteArrayOutputStream) outputStream; + if (MemoryUtils.BYTE_ARRAY_STREAM_WRAP_SUPPORTED + && outputStream.getClass() == ByteArrayOutputStream.class) { byte[] oldBytes = buf.getHeapMemory(); // Note: This should not be null. assert oldBytes != null; - MemoryUtils.wrap(byteArrayStream, buf); + MemoryUtils.wrap((ByteArrayOutputStream) outputStream, buf); function.accept(buf); - MemoryUtils.wrap(buf, byteArrayStream); + MemoryUtils.wrap(buf, (ByteArrayOutputStream) outputStream); buf.pointTo(oldBytes, 0, oldBytes.length); resetBuffer(); } else { diff --git a/java/fory-core/src/main/java/org/apache/fory/io/ForyByteArrayInputStream.java b/java/fory-core/src/main/java/org/apache/fory/io/ForyByteArrayInputStream.java deleted file mode 100644 index a622b78d36..0000000000 --- a/java/fory-core/src/main/java/org/apache/fory/io/ForyByteArrayInputStream.java +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.fory.io; - -import java.io.ByteArrayInputStream; -import org.apache.fory.util.Preconditions; - -/** A {@link ByteArrayInputStream} with public accessors for its protected state. */ -public class ForyByteArrayInputStream extends ByteArrayInputStream { - public ForyByteArrayInputStream(byte[] buffer) { - super(buffer); - } - - public ForyByteArrayInputStream(byte[] buffer, int offset, int length) { - super(buffer, offset, length); - } - - public byte[] getBuffer() { - return buf; - } - - public void setBuffer(byte[] buffer) { - buf = Preconditions.checkNotNull(buffer); - pos = 0; - mark = 0; - count = buffer.length; - } - - public void setBuffer(byte[] buffer, int offset, int length) { - Preconditions.checkNotNull(buffer); - Preconditions.checkArgument(offset >= 0 && length >= 0 && length <= buffer.length - offset); - buf = buffer; - pos = offset; - mark = offset; - count = offset + length; - } - - public int getPosition() { - return pos; - } - - public void setPosition(int position) { - Preconditions.checkArgument(position >= 0 && position <= count); - pos = position; - } - - public int getMark() { - return mark; - } - - public void setMark(int mark) { - Preconditions.checkArgument(mark >= 0 && mark <= count); - this.mark = mark; - } - - public int getCount() { - return count; - } - - public void setCount(int count) { - Preconditions.checkArgument(count >= 0 && count <= buf.length); - Preconditions.checkArgument(pos <= count && mark <= count); - this.count = count; - } -} diff --git a/java/fory-core/src/main/java/org/apache/fory/io/ForyByteArrayOutputStream.java b/java/fory-core/src/main/java/org/apache/fory/io/ForyByteArrayOutputStream.java deleted file mode 100644 index 572febfd78..0000000000 --- a/java/fory-core/src/main/java/org/apache/fory/io/ForyByteArrayOutputStream.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.fory.io; - -import java.io.ByteArrayOutputStream; -import org.apache.fory.util.Preconditions; - -/** A {@link ByteArrayOutputStream} with public accessors for its backing byte array and count. */ -public class ForyByteArrayOutputStream extends ByteArrayOutputStream { - public ForyByteArrayOutputStream() { - super(); - } - - public ForyByteArrayOutputStream(int size) { - super(size); - } - - public ForyByteArrayOutputStream(byte[] buffer) { - setBuffer(buffer); - } - - public ForyByteArrayOutputStream(byte[] buffer, int count) { - setBuffer(buffer); - setCount(count); - } - - public byte[] getBuffer() { - return buf; - } - - public void setBuffer(byte[] buffer) { - buf = Preconditions.checkNotNull(buffer); - if (count > buffer.length) { - count = buffer.length; - } - } - - public int getCount() { - return count; - } - - public void setCount(int count) { - Preconditions.checkArgument(count >= 0 && count <= buf.length); - this.count = count; - } -} diff --git a/java/fory-core/src/main/java/org/apache/fory/memory/MemoryUtils.java b/java/fory-core/src/main/java/org/apache/fory/memory/MemoryUtils.java index 6eef8aa81c..da42d6a9ac 100644 --- a/java/fory-core/src/main/java/org/apache/fory/memory/MemoryUtils.java +++ b/java/fory-core/src/main/java/org/apache/fory/memory/MemoryUtils.java @@ -19,14 +19,16 @@ package org.apache.fory.memory; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; import java.nio.ByteBuffer; -import org.apache.fory.io.ForyByteArrayInputStream; -import org.apache.fory.io.ForyByteArrayOutputStream; import org.apache.fory.platform.AndroidSupport; -import org.apache.fory.util.Preconditions; +import org.apache.fory.util.unsafe._JDKAccess; /** Memory utils for fory. */ public class MemoryUtils { + public static final boolean BYTE_ARRAY_STREAM_WRAP_SUPPORTED = + !AndroidSupport.IS_ANDROID && _JDKAccess.BYTE_ARRAY_STREAM_WRAP_SUPPORTED; public static MemoryBuffer buffer(int size) { return wrap(new byte[size]); @@ -71,40 +73,37 @@ public static MemoryBuffer wrap(ByteBuffer buffer) { } /** - * Wrap a {@link ForyByteArrayOutputStream} into a {@link MemoryBuffer}. The writerIndex of buffer - * will be the count of the stream. + * Wrap a {@link ByteArrayOutputStream} into a {@link MemoryBuffer}. The writerIndex of buffer + * will be the count of stream. */ - public static void wrap(ForyByteArrayOutputStream stream, MemoryBuffer buffer) { - Preconditions.checkNotNull(stream); - byte[] buf = stream.getBuffer(); - int count = stream.getCount(); - buffer.pointTo(buf, 0, buf.length); - buffer.writerIndex(count); + public static void wrap(ByteArrayOutputStream stream, MemoryBuffer buffer) { + checkByteArrayStreamWrap("ByteArrayOutputStream"); + _JDKAccess.wrap(stream, buffer); } /** - * Wrap a {@link MemoryBuffer} into a {@link ForyByteArrayOutputStream}. The count of the stream - * will be the writerIndex of buffer. + * Wrap a @link MemoryBuffer} into a {@link ByteArrayOutputStream}. The count of stream will be + * the writerIndex of buffer. */ - public static void wrap(MemoryBuffer buffer, ForyByteArrayOutputStream stream) { - Preconditions.checkNotNull(stream); - byte[] bytes = buffer.getHeapMemory(); - Preconditions.checkNotNull(bytes); - stream.setBuffer(bytes); - stream.setCount(buffer.writerIndex()); + public static void wrap(MemoryBuffer buffer, ByteArrayOutputStream stream) { + checkByteArrayStreamWrap("ByteArrayOutputStream"); + _JDKAccess.wrap(buffer, stream); } /** - * Wrap a {@link ForyByteArrayInputStream} into a {@link MemoryBuffer}. The readerIndex of buffer - * will be the position of the stream. + * Wrap a {@link ByteArrayInputStream} into a {@link MemoryBuffer}. The readerIndex of buffer will + * be the pos of stream. */ - public static void wrap(ForyByteArrayInputStream stream, MemoryBuffer buffer) { - Preconditions.checkNotNull(stream); - byte[] buf = stream.getBuffer(); - int count = stream.getCount(); - int pos = stream.getPosition(); - buffer.pointTo(buf, 0, count); - buffer.readerIndex(pos); + public static void wrap(ByteArrayInputStream stream, MemoryBuffer buffer) { + checkByteArrayStreamWrap("ByteArrayInputStream"); + _JDKAccess.wrap(stream, buffer); + } + + private static void checkByteArrayStreamWrap(String streamType) { + if (!BYTE_ARRAY_STREAM_WRAP_SUPPORTED) { + throw new UnsupportedOperationException( + streamType + " direct wrapping is not supported on this platform"); + } } private static MemoryBuffer copyToHeapBuffer(ByteBuffer buffer) { diff --git a/java/fory-core/src/main/java/org/apache/fory/util/unsafe/_JDKAccess.java b/java/fory-core/src/main/java/org/apache/fory/util/unsafe/_JDKAccess.java index d41b302679..bd2f143666 100644 --- a/java/fory-core/src/main/java/org/apache/fory/util/unsafe/_JDKAccess.java +++ b/java/fory-core/src/main/java/org/apache/fory/util/unsafe/_JDKAccess.java @@ -19,6 +19,8 @@ package org.apache.fory.util.unsafe; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; import java.lang.invoke.CallSite; import java.lang.invoke.LambdaMetafactory; import java.lang.invoke.MethodHandle; @@ -39,6 +41,7 @@ import java.util.function.ToLongFunction; import org.apache.fory.collection.ClassValueCache; import org.apache.fory.collection.Tuple2; +import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.platform.GraalvmSupport; import org.apache.fory.platform.JdkVersion; import org.apache.fory.type.TypeUtils; @@ -56,6 +59,7 @@ public class _JDKAccess { // CHECKSTYLE.ON:TypeName public static final boolean IS_OPEN_J9; public static final Unsafe UNSAFE; + public static final boolean BYTE_ARRAY_STREAM_WRAP_SUPPORTED; public static final Class _INNER_UNSAFE_CLASS; public static final Object _INNER_UNSAFE; @@ -71,6 +75,7 @@ public class _JDKAccess { throw new UnsupportedOperationException("Unsafe is not supported in this platform."); } UNSAFE = unsafe; + BYTE_ARRAY_STREAM_WRAP_SUPPORTED = true; if (JdkVersion.MAJOR_VERSION >= 11) { try { Field theInternalUnsafeField = Unsafe.class.getDeclaredField("theInternalUnsafe"); @@ -100,6 +105,55 @@ public static Lookup _trustedLookup(Class objectClass) { return lookupCache.get(objectClass, () -> _Lookup._trustedLookup(objectClass)); } + // Lazy load offsets and keep the access shape in one class so the JDK25 multi-release + // replacement can change these methods without touching MemoryUtils callers. + private static class ByteArrayStreamFields { + private static final long BAS_BUF_BUF; + private static final long BAS_BUF_COUNT; + private static final long BIS_BUF_BUF; + private static final long BIS_BUF_POS; + private static final long BIS_BUF_COUNT; + + static { + try { + BAS_BUF_BUF = UNSAFE.objectFieldOffset(ByteArrayOutputStream.class.getDeclaredField("buf")); + BAS_BUF_COUNT = + UNSAFE.objectFieldOffset(ByteArrayOutputStream.class.getDeclaredField("count")); + BIS_BUF_BUF = UNSAFE.objectFieldOffset(ByteArrayInputStream.class.getDeclaredField("buf")); + BIS_BUF_POS = UNSAFE.objectFieldOffset(ByteArrayInputStream.class.getDeclaredField("pos")); + BIS_BUF_COUNT = + UNSAFE.objectFieldOffset(ByteArrayInputStream.class.getDeclaredField("count")); + } catch (NoSuchFieldException e) { + throw new RuntimeException(e); + } + } + } + + public static void wrap(ByteArrayOutputStream stream, MemoryBuffer buffer) { + Preconditions.checkNotNull(stream); + byte[] buf = (byte[]) UNSAFE.getObject(stream, ByteArrayStreamFields.BAS_BUF_BUF); + int count = UNSAFE.getInt(stream, ByteArrayStreamFields.BAS_BUF_COUNT); + buffer.pointTo(buf, 0, buf.length); + buffer.writerIndex(count); + } + + public static void wrap(MemoryBuffer buffer, ByteArrayOutputStream stream) { + Preconditions.checkNotNull(stream); + byte[] bytes = buffer.getHeapMemory(); + Preconditions.checkNotNull(bytes); + UNSAFE.putObject(stream, ByteArrayStreamFields.BAS_BUF_BUF, bytes); + UNSAFE.putInt(stream, ByteArrayStreamFields.BAS_BUF_COUNT, buffer.writerIndex()); + } + + public static void wrap(ByteArrayInputStream stream, MemoryBuffer buffer) { + Preconditions.checkNotNull(stream); + byte[] buf = (byte[]) UNSAFE.getObject(stream, ByteArrayStreamFields.BIS_BUF_BUF); + int count = UNSAFE.getInt(stream, ByteArrayStreamFields.BIS_BUF_COUNT); + int pos = UNSAFE.getInt(stream, ByteArrayStreamFields.BIS_BUF_POS); + buffer.pointTo(buf, 0, count); + buffer.readerIndex(pos); + } + public static T tryMakeFunction( Lookup lookup, MethodHandle handle, Class functionInterface) { try { diff --git a/java/fory-core/src/test/java/org/apache/fory/StreamTest.java b/java/fory-core/src/test/java/org/apache/fory/StreamTest.java index 40860d8b72..e25b486493 100644 --- a/java/fory-core/src/test/java/org/apache/fory/StreamTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/StreamTest.java @@ -38,7 +38,6 @@ import java.util.List; import java.util.Map; import org.apache.fory.exception.DeserializationException; -import org.apache.fory.io.ForyByteArrayOutputStream; import org.apache.fory.io.ForyInputStream; import org.apache.fory.io.ForyReadableChannel; import org.apache.fory.io.ForyStreamReader; @@ -134,11 +133,6 @@ public void testBufferReset() { assertEquals(o, new byte[1000 * 1000]); assertEquals(fory.deserialize(bas.toByteArray()), new byte[1000 * 1000]); - ForyByteArrayOutputStream foryBas = new ForyByteArrayOutputStream(); - fory.serialize(foryBas, "fory-stream"); - checkBuffer(fory); - assertEquals(fory.deserialize(foryBas.toByteArray()), "fory-stream"); - bas.reset(); fory.serialize(bas, new byte[1000 * 1000]); checkBuffer(fory); diff --git a/java/fory-core/src/test/java/org/apache/fory/memory/MemoryBufferTest.java b/java/fory-core/src/test/java/org/apache/fory/memory/MemoryBufferTest.java index 5d035e30cb..9ee62e3876 100644 --- a/java/fory-core/src/test/java/org/apache/fory/memory/MemoryBufferTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/memory/MemoryBufferTest.java @@ -23,6 +23,7 @@ import static org.testng.Assert.assertThrows; import static org.testng.Assert.assertTrue; +import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; @@ -30,8 +31,6 @@ import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.util.Random; -import org.apache.fory.io.ForyByteArrayInputStream; -import org.apache.fory.io.ForyByteArrayOutputStream; import org.apache.fory.platform.AndroidSupport; import org.testng.Assert; import org.testng.annotations.Test; @@ -84,22 +83,24 @@ public void testBufferWrite() { } @Test - public void testForyByteArrayStreamWrap() { - ForyByteArrayOutputStream outputStream = new ForyByteArrayOutputStream(8); + public void testByteArrayStreamWrap() { + if (!MemoryUtils.BYTE_ARRAY_STREAM_WRAP_SUPPORTED) { + return; + } + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(8); outputStream.write(new byte[] {1, 2, 3}, 0, 3); MemoryBuffer buffer = MemoryUtils.buffer(1); MemoryUtils.wrap(outputStream, buffer); - assertEquals(buffer.getHeapMemory(), outputStream.getBuffer()); assertEquals(buffer.writerIndex(), 3); + assertEquals(buffer.getByte(0), (byte) 1); buffer.writeByte((byte) 4); MemoryUtils.wrap(buffer, outputStream); - assertEquals(outputStream.getCount(), 4); - assertEquals(outputStream.getBuffer()[3], (byte) 4); + assertEquals(outputStream.size(), 4); + assertEquals(outputStream.toByteArray(), new byte[] {1, 2, 3, 4}); - ForyByteArrayInputStream inputStream = new ForyByteArrayInputStream(new byte[] {5, 6, 7}); + ByteArrayInputStream inputStream = new ByteArrayInputStream(new byte[] {5, 6, 7}); assertEquals(inputStream.read(), 5); MemoryUtils.wrap(inputStream, buffer); - assertEquals(buffer.getHeapMemory(), inputStream.getBuffer()); assertEquals(buffer.readerIndex(), 1); assertEquals(buffer.readByte(), (byte) 6); } diff --git a/java/fory-core/src/test/java/org/apache/fory/serializer/AndroidDynamicFeatureTest.java b/java/fory-core/src/test/java/org/apache/fory/serializer/AndroidDynamicFeatureTest.java index 3be0ea6756..5e251c2d06 100644 --- a/java/fory-core/src/test/java/org/apache/fory/serializer/AndroidDynamicFeatureTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/serializer/AndroidDynamicFeatureTest.java @@ -19,6 +19,7 @@ package org.apache.fory.serializer; +import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; @@ -38,9 +39,6 @@ import org.apache.fory.Fory; import org.apache.fory.context.ReadContext; import org.apache.fory.context.WriteContext; -import org.apache.fory.io.ForyByteArrayInputStream; -import org.apache.fory.io.ForyByteArrayOutputStream; -import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.memory.MemoryUtils; import org.apache.fory.platform.AndroidSupport; import org.apache.fory.resolver.TypeResolver; @@ -89,7 +87,7 @@ public static void main(String[] args) { LambdaSerializer.STUB_LAMBDA_CLASS == LambdaSerializer.ReplaceStub.class, "Android must not create a runtime lambda stub class"); verifyReflectiveGetter(); - verifyMemoryUtilsStreamWraps(); + verifyMemoryUtilsStreamWrapGuards(); verifyXlangUnion(); verifyFory(false); @@ -174,22 +172,21 @@ private static void verifyOutputStreamSerialization(Fory fory) { checkEquals(fory.deserialize(outputStream.toByteArray()), value, "OutputStream round trip"); } - private static void verifyMemoryUtilsStreamWraps() { - MemoryBuffer buffer = MemoryUtils.buffer(8); - ForyByteArrayOutputStream outputStream = new ForyByteArrayOutputStream(8); - outputStream.write(new byte[] {1, 2}, 0, 2); - MemoryUtils.wrap(outputStream, buffer); - checkEquals(buffer.writerIndex(), 2, "Output stream writer index"); - buffer.writeByte((byte) 3); - MemoryUtils.wrap(buffer, outputStream); - checkEquals(outputStream.getCount(), 3, "Output stream count"); - checkEquals(outputStream.getBuffer()[2], (byte) 3, "Output stream buffer"); - - ForyByteArrayInputStream inputStream = new ForyByteArrayInputStream(new byte[] {4, 5, 6}); - checkEquals(inputStream.read(), 4, "Input stream first byte"); - MemoryUtils.wrap(inputStream, buffer); - checkEquals(buffer.readerIndex(), 1, "Input stream reader index"); - checkEquals(buffer.readByte(), (byte) 5, "Input stream buffer"); + private static void verifyMemoryUtilsStreamWrapGuards() { + check( + !MemoryUtils.BYTE_ARRAY_STREAM_WRAP_SUPPORTED, + "Android must report byte-array stream wrapping unsupported"); + expectUnsupportedAndroidWrap( + () -> MemoryUtils.wrap(new ByteArrayOutputStream(), MemoryUtils.buffer(8)), + "ByteArrayOutputStream direct wrapping"); + expectUnsupportedAndroidWrap( + () -> MemoryUtils.wrap(MemoryUtils.buffer(8), new ByteArrayOutputStream()), + "ByteArrayOutputStream direct wrapping"); + expectUnsupportedAndroidWrap( + () -> + MemoryUtils.wrap( + new ByteArrayInputStream(new byte[] {1, 2, 3}), MemoryUtils.buffer(8)), + "ByteArrayInputStream direct wrapping"); } private static void verifyXlangUnion() { @@ -263,6 +260,17 @@ private static void expectUnsupported(Serializer serializer) { } } + private static void expectUnsupportedAndroidWrap(Runnable operation, String messageFragment) { + try { + operation.run(); + throw new AssertionError("Expected Android unsafe stream wrapping to fail"); + } catch (UnsupportedOperationException expected) { + check( + expected.getMessage().contains(messageFragment), + "Unexpected unsupported message: " + expected.getMessage()); + } + } + private static void check(boolean value, String message) { if (!value) { throw new AssertionError(message); From 8361bd4396b7174bbd96eea62ffbf111b0047ae4 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sat, 23 May 2026 22:29:21 +0800 Subject: [PATCH 09/69] remove unsafe fields sort --- .../java/org/apache/fory/meta/TypeDef.java | 27 ------------------- .../org/apache/fory/meta/TypeDefTest.java | 11 -------- 2 files changed, 38 deletions(-) diff --git a/java/fory-core/src/main/java/org/apache/fory/meta/TypeDef.java b/java/fory-core/src/main/java/org/apache/fory/meta/TypeDef.java index 4a72cb78d0..86a82e3e37 100644 --- a/java/fory-core/src/main/java/org/apache/fory/meta/TypeDef.java +++ b/java/fory-core/src/main/java/org/apache/fory/meta/TypeDef.java @@ -19,14 +19,12 @@ package org.apache.fory.meta; -import static org.apache.fory.meta.NativeTypeDefEncoder.buildDescriptors; import java.io.ObjectStreamClass; import java.io.Serializable; import java.lang.reflect.Field; import java.util.ArrayList; import java.util.Collection; -import java.util.Comparator; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -38,7 +36,6 @@ import org.apache.fory.logging.Logger; import org.apache.fory.logging.LoggerFactory; import org.apache.fory.memory.MemoryBuffer; -import org.apache.fory.platform.UnsafeOps; import org.apache.fory.reflect.ReflectionUtils; import org.apache.fory.resolver.ClassResolver; import org.apache.fory.resolver.SharedRegistry; @@ -78,30 +75,6 @@ public class TypeDef implements Serializable { static final int META_SIZE_MASKS = 0xff; static final int NUM_HASH_BITS = 52; - // TODO use field offset to sort field, which will hit l1-cache more. Since - // `objectFieldOffset` is not part of jvm-specification, it may change between different jdk - // vendor. But the deserialization peer use the class definition to create deserializer, it's OK - // even field offset or fields order change between jvm process. - public static final Comparator FIELD_COMPARATOR = - (f1, f2) -> { - long offset1 = UnsafeOps.objectFieldOffset(f1); - long offset2 = UnsafeOps.objectFieldOffset(f2); - long diff = offset1 - offset2; - if (diff != 0) { - return (int) diff; - } else { - if (!f1.equals(f2)) { - LOG.warn( - "Field {} has same offset with {}, please an issue with jdk info to fory", f1, f2); - } - int compare = f1.getDeclaringClass().getName().compareTo(f2.getName()); - if (compare != 0) { - return compare; - } - return f1.getName().compareTo(f2.getName()); - } - }; - private final ClassSpec classSpec; private final List fieldsInfo; // Unique id for class def. If class def are same between processes, then the id will diff --git a/java/fory-core/src/test/java/org/apache/fory/meta/TypeDefTest.java b/java/fory-core/src/test/java/org/apache/fory/meta/TypeDefTest.java index 19e5378c3f..6367a8625c 100644 --- a/java/fory-core/src/test/java/org/apache/fory/meta/TypeDefTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/meta/TypeDefTest.java @@ -85,17 +85,6 @@ static class ContainerClass extends TestFieldsOrderClass1 { private Map map3; } - @Test - public void testFieldsOrder() { - List fieldList = new ArrayList<>(); - Collections.addAll(fieldList, TestFieldsOrderClass1.class.getDeclaredFields()); - Collections.addAll(fieldList, TestFieldsOrderClass2.class.getDeclaredFields()); - TreeSet sorted = new TreeSet<>(TypeDef.FIELD_COMPARATOR); - sorted.addAll(fieldList); - assertEquals(fieldList.size(), sorted.size()); - fieldList.sort(TypeDef.FIELD_COMPARATOR); - } - @Test public void testTypeDefSerialization() throws NoSuchFieldException { Fory fory = Fory.builder().withXlang(false).withMetaShare(true).build(); From c56356545d9ed0f2650eb547656233e73e1c4710 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sat, 23 May 2026 22:54:24 +0800 Subject: [PATCH 10/69] refactor(java): route private field access through accessors --- .../src/main/java/org/apache/fory/Fory.java | 2 +- .../org/apache/fory/memory/MemoryUtils.java | 9 +- .../apache/fory/reflect/FieldAccessor.java | 57 +++++++++ .../apache/fory/reflect/ReflectionUtils.java | 32 +---- .../fory/serializer/ExceptionSerializers.java | 19 ++- .../apache/fory/serializer/FieldGroups.java | 2 +- .../fory/serializer/JdkProxySerializer.java | 15 +-- .../fory/serializer/StringSerializer.java | 120 +++++++----------- .../collection/CollectionSerializers.java | 49 ++++--- .../serializer/collection/MapSerializers.java | 15 +-- .../collection/SubListSerializers.java | 33 ++--- .../collection/SynchronizedSerializers.java | 54 ++++---- .../collection/UnmodifiableSerializers.java | 55 ++++---- .../scala/SingletonCollectionSerializer.java | 14 +- .../scala/SingletonMapSerializer.java | 14 +- .../scala/SingletonObjectSerializer.java | 14 +- .../apache/fory/util/unsafe/_JDKAccess.java | 77 ++++++++++- .../apache/fory/codegen/JaninoUtilsTest.java | 9 +- .../apache/fory/memory/MemoryBufferTest.java | 2 +- .../serializer/AndroidDynamicFeatureTest.java | 4 +- .../fory/serializer/StringSerializerTest.java | 6 +- .../SynchronizedSerializersTest.java | 28 ++-- .../UnmodifiableSerializersTest.java | 27 ++-- 23 files changed, 352 insertions(+), 305 deletions(-) diff --git a/java/fory-core/src/main/java/org/apache/fory/Fory.java b/java/fory-core/src/main/java/org/apache/fory/Fory.java index 140d83f28c..ed770247a3 100644 --- a/java/fory-core/src/main/java/org/apache/fory/Fory.java +++ b/java/fory-core/src/main/java/org/apache/fory/Fory.java @@ -563,7 +563,7 @@ public T copy(T obj) { private void serializeToStream(OutputStream outputStream, Consumer function) { MemoryBuffer buf = getBuffer(); - if (MemoryUtils.BYTE_ARRAY_STREAM_WRAP_SUPPORTED + if (MemoryUtils.JDK_INTERNAL_FIELD_ACCESS && outputStream.getClass() == ByteArrayOutputStream.class) { byte[] oldBytes = buf.getHeapMemory(); // Note: This should not be null. assert oldBytes != null; diff --git a/java/fory-core/src/main/java/org/apache/fory/memory/MemoryUtils.java b/java/fory-core/src/main/java/org/apache/fory/memory/MemoryUtils.java index da42d6a9ac..9194937c4e 100644 --- a/java/fory-core/src/main/java/org/apache/fory/memory/MemoryUtils.java +++ b/java/fory-core/src/main/java/org/apache/fory/memory/MemoryUtils.java @@ -27,8 +27,11 @@ /** Memory utils for fory. */ public class MemoryUtils { - public static final boolean BYTE_ARRAY_STREAM_WRAP_SUPPORTED = - !AndroidSupport.IS_ANDROID && _JDKAccess.BYTE_ARRAY_STREAM_WRAP_SUPPORTED; + // JDK25+ internal-field access must be backed by supported access in the multi-release classes. + // When a JDK25+ path needs java.nio private fields, the JVM must be launched with: + // --add-opens=java.base/java.nio=org.apache.fory.core + public static final boolean JDK_INTERNAL_FIELD_ACCESS = + !AndroidSupport.IS_ANDROID && _JDKAccess.JDK_INTERNAL_FIELD_ACCESS; public static MemoryBuffer buffer(int size) { return wrap(new byte[size]); @@ -100,7 +103,7 @@ public static void wrap(ByteArrayInputStream stream, MemoryBuffer buffer) { } private static void checkByteArrayStreamWrap(String streamType) { - if (!BYTE_ARRAY_STREAM_WRAP_SUPPORTED) { + if (!JDK_INTERNAL_FIELD_ACCESS) { throw new UnsupportedOperationException( streamType + " direct wrapping is not supported on this platform"); } diff --git a/java/fory-core/src/main/java/org/apache/fory/reflect/FieldAccessor.java b/java/fory-core/src/main/java/org/apache/fory/reflect/FieldAccessor.java index eda245f973..2ff89fc529 100644 --- a/java/fory-core/src/main/java/org/apache/fory/reflect/FieldAccessor.java +++ b/java/fory-core/src/main/java/org/apache/fory/reflect/FieldAccessor.java @@ -24,6 +24,7 @@ import java.lang.reflect.Field; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; +import java.lang.reflect.Modifier; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.function.Function; @@ -191,6 +192,7 @@ public Object getGetter() { } public static FieldAccessor createAccessor(Field field) { + Preconditions.checkArgument(!Modifier.isStatic(field.getModifiers()), field); if (RecordUtils.isRecord(field.getDeclaringClass())) { if (AndroidSupport.IS_ANDROID || GraalvmSupport.IN_GRAALVM_NATIVE_IMAGE) { return new ReflectiveRecordFieldAccessor(field); @@ -251,6 +253,15 @@ public static FieldAccessor createAccessor(Field field) { } } + public static FieldAccessor createStaticAccessor(Field field) { + Preconditions.checkArgument(Modifier.isStatic(field.getModifiers()), field); + if (AndroidSupport.IS_ANDROID) { + field.setAccessible(true); + return new ReflectiveStaticFieldAccessor(field); + } + return new StaticObjectAccessor(field); + } + static final class ReflectiveRecordFieldAccessor extends FieldGetter { private final Method accessor; @@ -720,6 +731,52 @@ public Object get(Object obj) { } } + static final class ReflectiveStaticFieldAccessor extends FieldAccessor { + ReflectiveStaticFieldAccessor(Field field) { + super(field, -1); + } + + @Override + public Object get(Object obj) { + try { + return field.get(null); + } catch (IllegalAccessException | IllegalArgumentException e) { + throw new ForyException("Failed to read static field reflectively: " + field, e); + } + } + + @Override + public void set(Object obj, Object value) { + try { + field.set(null, value); + } catch (IllegalAccessException | IllegalArgumentException e) { + throw new ForyException("Failed to write static field reflectively: " + field, e); + } + } + } + + static final class StaticObjectAccessor extends FieldAccessor { + private final Object base; + private final long offset; + + StaticObjectAccessor(Field field) { + super(field, -1); + Preconditions.checkArgument(!TypeUtils.isPrimitive(field.getType())); + base = UnsafeOps.UNSAFE.staticFieldBase(field); + offset = UnsafeOps.UNSAFE.staticFieldOffset(field); + } + + @Override + public Object get(Object obj) { + return UnsafeOps.getObject(base, offset); + } + + @Override + public void set(Object obj, Object value) { + UnsafeOps.putObject(base, offset, value); + } + } + static final class GeneratedAccessor extends FieldAccessor { private static final ClassValueCache>> cache = ClassValueCache.newClassKeyCache(8); diff --git a/java/fory-core/src/main/java/org/apache/fory/reflect/ReflectionUtils.java b/java/fory-core/src/main/java/org/apache/fory/reflect/ReflectionUtils.java index 2af7dd0d0b..e09b3361bd 100644 --- a/java/fory-core/src/main/java/org/apache/fory/reflect/ReflectionUtils.java +++ b/java/fory-core/src/main/java/org/apache/fory/reflect/ReflectionUtils.java @@ -499,30 +499,13 @@ public static void setObjectFieldValue(Object obj, String fieldName, Object valu public static void setObjectFieldValue(Object obj, Field field, Object value) { Preconditions.checkArgument( !field.getType().isPrimitive(), "Field %s is primitive type", field); - if (AndroidSupport.IS_ANDROID) { - try { - field.setAccessible(true); - field.set(obj, value); - return; - } catch (IllegalAccessException | IllegalArgumentException e) { - throw new ForyException("Failed to write object field reflectively: " + field, e); - } - } - UnsafeOps.putObject(obj, UnsafeOps.objectFieldOffset(field), value); + FieldAccessor.createAccessor(field).putObject(obj, value); } public static T getObjectFieldValue(Object obj, Field field) { Preconditions.checkArgument( !field.getType().isPrimitive(), "Field %s is primitive type", field); - if (AndroidSupport.IS_ANDROID) { - try { - field.setAccessible(true); - return (T) field.get(obj); - } catch (IllegalAccessException | IllegalArgumentException e) { - throw new ForyException("Failed to read object field reflectively: " + field, e); - } - } - return (T) UnsafeOps.getObject(obj, UnsafeOps.objectFieldOffset(field)); + return (T) FieldAccessor.createAccessor(field).getObject(obj); } /** @@ -538,16 +521,7 @@ public static Object getObjectFieldValue(Object obj, String fieldName) { Field field = cls.getDeclaredField(fieldName); Preconditions.checkArgument( !field.getType().isPrimitive(), "Field %s is primitive type", field); - if (AndroidSupport.IS_ANDROID) { - try { - field.setAccessible(true); - return field.get(obj); - } catch (IllegalAccessException | IllegalArgumentException e) { - throw new ForyException("Failed to read object field reflectively: " + field, e); - } - } - long fieldOffset = UnsafeOps.objectFieldOffset(field); - return UnsafeOps.getObject(obj, fieldOffset); + return FieldAccessor.createAccessor(field).getObject(obj); // CHECKSTYLE.OFF:EmptyCatchBlock } catch (NoSuchFieldException ignored) { } diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/ExceptionSerializers.java b/java/fory-core/src/main/java/org/apache/fory/serializer/ExceptionSerializers.java index dc22b47edb..e648da8716 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/ExceptionSerializers.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/ExceptionSerializers.java @@ -38,10 +38,11 @@ import org.apache.fory.context.WriteContext; import org.apache.fory.exception.ForyException; import org.apache.fory.memory.MemoryBuffer; +import org.apache.fory.memory.MemoryUtils; import org.apache.fory.meta.TypeDef; import org.apache.fory.platform.AndroidSupport; import org.apache.fory.platform.GraalvmSupport; -import org.apache.fory.platform.UnsafeOps; +import org.apache.fory.reflect.FieldAccessor; import org.apache.fory.reflect.ObjectCreator; import org.apache.fory.reflect.ObjectCreators; import org.apache.fory.reflect.ReflectionUtils; @@ -69,11 +70,11 @@ public ExceptionSerializer(TypeResolver typeResolver, Class type) { this.typeResolver = typeResolver; messageConstructor = getOptionalMessageConstructor(type); objectCreator = - messageConstructor == null && !AndroidSupport.IS_ANDROID + messageConstructor == null && MemoryUtils.JDK_INTERNAL_FIELD_ACCESS ? createThrowableObjectCreator(type) : null; slotsSerializers = buildSlotsSerializers(typeResolver, type); - if (AndroidSupport.IS_ANDROID + if (!MemoryUtils.JDK_INTERNAL_FIELD_ACCESS && isJdkThrowable(type) && hasSubclassFields(slotsSerializers)) { throw new ForyException( @@ -110,7 +111,7 @@ public void write(WriteContext writeContext, T value) { public T read(ReadContext readContext) { Serializer[] slotsSerializers = getSlotsSerializers(); StackTraceElement[] stackTrace = (StackTraceElement[]) readContext.readRef(); - if (AndroidSupport.IS_ANDROID) { + if (!MemoryUtils.JDK_INTERNAL_FIELD_ACCESS) { return readAndroidThrowableWithoutDetailMessageField( readContext, stackTrace, slotsSerializers); } @@ -120,7 +121,7 @@ public T read(ReadContext readContext) { String detailMessage = readContext.readStringRef(); List suppressedExceptions = readSuppressedExceptions(readContext); skipExtraFields(readContext); - UnsafeOps.putObject(obj, ThrowableOffsets.DETAIL_MESSAGE_FIELD_OFFSET, detailMessage); + ThrowableAccessors.DETAIL_MESSAGE_ACCESSOR.putObject(obj, detailMessage); if (stackTrace != null) { obj.setStackTrace(stackTrace); } @@ -532,15 +533,13 @@ private static boolean containsPendingThrowable(Throwable throwable, Set { private static final byte UTF8 = 2; private static final int DEFAULT_BUFFER_SIZE = 1024; - // Make offset compatible with graalvm native image. - private static final long STRING_VALUE_FIELD_OFFSET; private static final boolean STRING_HAS_COUNT_OFFSET; - private static final long STRING_COUNT_FIELD_OFFSET; - private static final long STRING_OFFSET_FIELD_OFFSET; - - private static class Offset { - // Make offset compatible with graalvm native image. - private static final long STRING_CODER_FIELD_OFFSET; - - static { - if (AndroidSupport.IS_ANDROID) { - STRING_CODER_FIELD_OFFSET = -1; - } else { - try { - STRING_CODER_FIELD_OFFSET = - UnsafeOps.objectFieldOffset(String.class.getDeclaredField("coder")); - } catch (NoSuchFieldException e) { - throw new RuntimeException(e); - } - } - } - } static { - if (AndroidSupport.IS_ANDROID) { + if (!jdkInternalFieldAccess()) { STRING_VALUE_FIELD_IS_CHARS = false; STRING_VALUE_FIELD_IS_BYTES = false; - STRING_VALUE_FIELD_OFFSET = -1; STRING_HAS_COUNT_OFFSET = false; - STRING_COUNT_FIELD_OFFSET = -1; - STRING_OFFSET_FIELD_OFFSET = -1; } else { - Field valueField = ReflectionUtils.getFieldNullable(String.class, "value"); - // Java8 string - STRING_VALUE_FIELD_IS_CHARS = valueField != null && valueField.getType() == char[].class; - // Java11 string - STRING_VALUE_FIELD_IS_BYTES = valueField != null && valueField.getType() == byte[].class; - try { - // Make offset compatible with graalvm native image. - STRING_VALUE_FIELD_OFFSET = - UnsafeOps.objectFieldOffset(String.class.getDeclaredField("value")); - } catch (NoSuchFieldException e) { - throw new RuntimeException(e); - } - Field countField = ReflectionUtils.getFieldNullable(String.class, "count"); - Field offsetField = ReflectionUtils.getFieldNullable(String.class, "offset"); - if (countField != null || offsetField != null) { - Preconditions.checkArgument( - countField != null && offsetField != null, "Current jdk not supported"); - Preconditions.checkArgument( - countField.getType() == int.class && offsetField.getType() == int.class, - "Current jdk not supported"); - STRING_HAS_COUNT_OFFSET = true; - STRING_COUNT_FIELD_OFFSET = UnsafeOps.objectFieldOffset(countField); - STRING_OFFSET_FIELD_OFFSET = UnsafeOps.objectFieldOffset(offsetField); - } else { - STRING_HAS_COUNT_OFFSET = false; - STRING_COUNT_FIELD_OFFSET = -1; - STRING_OFFSET_FIELD_OFFSET = -1; - } + STRING_VALUE_FIELD_IS_CHARS = _JDKAccess.STRING_VALUE_FIELD_IS_CHARS; + STRING_VALUE_FIELD_IS_BYTES = _JDKAccess.STRING_VALUE_FIELD_IS_BYTES; + STRING_HAS_COUNT_OFFSET = _JDKAccess.STRING_HAS_COUNT_OFFSET; } } + private static boolean jdkInternalFieldAccess() { + return !AndroidSupport.IS_ANDROID && _JDKAccess.JDK_INTERNAL_FIELD_ACCESS; + } + private final boolean compressString; private final boolean writeNumUtf16BytesForUtf8Encoding; private final boolean xlang; @@ -363,7 +315,7 @@ public String readCompressedCharsString(MemoryBuffer buffer) { // Invoked by fory JIT public void writeString(MemoryBuffer buffer, String value) { - if (AndroidSupport.IS_ANDROID) { + if (!jdkInternalFieldAccess()) { writeStringSlow(buffer, value); return; } @@ -397,7 +349,7 @@ private void writeJava8String(MemoryBuffer buffer, String value) { // Invoked by fory JIT public String readString(MemoryBuffer buffer) { - if (AndroidSupport.IS_ANDROID) { + if (!jdkInternalFieldAccess()) { return readStringSlow(buffer); } if (STRING_VALUE_FIELD_IS_BYTES) { @@ -474,10 +426,26 @@ private static boolean isLatin(char[] chars) { return true; } + private static Object getStringValue(String value) { + return _JDKAccess.getStringValue(value); + } + + private static byte getStringCoder(String value) { + return _JDKAccess.getStringCoder(value); + } + + private static int getStringOffset(String value) { + return _JDKAccess.getStringOffset(value); + } + + private static int getStringCount(String value) { + return _JDKAccess.getStringCount(value); + } + @CodegenInvoke public void writeCompressedBytesString(MemoryBuffer buffer, String value) { - final byte[] bytes = (byte[]) UnsafeOps.getObject(value, STRING_VALUE_FIELD_OFFSET); - final byte coder = UnsafeOps.getByte(value, Offset.STRING_CODER_FIELD_OFFSET); + final byte[] bytes = (byte[]) getStringValue(value); + final byte coder = getStringCoder(value); if (coder == LATIN1 || bestCoder(bytes) == UTF16) { writeBytesString(buffer, coder, bytes); } else { @@ -491,7 +459,7 @@ public void writeCompressedBytesString(MemoryBuffer buffer, String value) { @CodegenInvoke public void writeCompressedCharsString(MemoryBuffer buffer, String value) { - final char[] chars = (char[]) UnsafeOps.getObject(value, STRING_VALUE_FIELD_OFFSET); + final char[] chars = (char[]) getStringValue(value); final byte coder = bestCoder(chars); if (coder == LATIN1) { writeCharsLatin1(buffer, chars, chars.length); @@ -508,9 +476,9 @@ public void writeCompressedCharsString(MemoryBuffer buffer, String value) { @CodegenInvoke public void writeCompressedCharsStringWithOffset(MemoryBuffer buffer, String value) { - final char[] chars = (char[]) UnsafeOps.getObject(value, STRING_VALUE_FIELD_OFFSET); - final int offset = UnsafeOps.getInt(value, STRING_OFFSET_FIELD_OFFSET); - final int count = UnsafeOps.getInt(value, STRING_COUNT_FIELD_OFFSET); + final char[] chars = (char[]) getStringValue(value); + final int offset = getStringOffset(value); + final int count = getStringCount(value); final byte coder = SlicedStringUtil.bestCoder(chars, offset, count); if (coder == LATIN1) { SlicedStringUtil.writeCharsLatin1WithOffset(this, buffer, chars, offset, count); @@ -527,8 +495,8 @@ public void writeCompressedCharsStringWithOffset(MemoryBuffer buffer, String val @CodegenInvoke public static void writeBytesString(MemoryBuffer buffer, String value) { - byte[] bytes = (byte[]) UnsafeOps.getObject(value, STRING_VALUE_FIELD_OFFSET); - byte coder = UnsafeOps.getByte(value, Offset.STRING_CODER_FIELD_OFFSET); + byte[] bytes = (byte[]) getStringValue(value); + byte coder = getStringCoder(value); writeBytesString(buffer, coder, bytes); } @@ -568,7 +536,7 @@ public static void writeBytesString(MemoryBuffer buffer, byte coder, byte[] byte @CodegenInvoke public void writeCharsString(MemoryBuffer buffer, String value) { - final char[] chars = (char[]) UnsafeOps.getObject(value, STRING_VALUE_FIELD_OFFSET); + final char[] chars = (char[]) getStringValue(value); if (StringUtils.isLatin(chars)) { writeCharsLatin1(buffer, chars, chars.length); } else { @@ -578,9 +546,9 @@ public void writeCharsString(MemoryBuffer buffer, String value) { @CodegenInvoke public void writeCharsStringWithOffset(MemoryBuffer buffer, String value) { - final char[] chars = (char[]) UnsafeOps.getObject(value, STRING_VALUE_FIELD_OFFSET); - final int offset = UnsafeOps.getInt(value, STRING_OFFSET_FIELD_OFFSET); - final int count = UnsafeOps.getInt(value, STRING_COUNT_FIELD_OFFSET); + final char[] chars = (char[]) getStringValue(value); + final int offset = getStringOffset(value); + final int count = getStringCount(value); if (SlicedStringUtil.isLatin(chars, offset, count)) { SlicedStringUtil.writeCharsLatin1WithOffset(this, buffer, chars, offset, count); } else { @@ -918,16 +886,16 @@ private void writeBytesUTF8PerfOptimized(MemoryBuffer buffer, byte[] bytes) { } private static final MethodHandles.Lookup STRING_LOOK_UP = - AndroidSupport.IS_ANDROID ? null : _JDKAccess._trustedLookup(String.class); + jdkInternalFieldAccess() ? _JDKAccess._trustedLookup(String.class) : null; private static final BiFunction CHARS_STRING_ZERO_COPY_CTR = - AndroidSupport.IS_ANDROID ? null : getCharsStringZeroCopyCtr(); + jdkInternalFieldAccess() ? getCharsStringZeroCopyCtr() : null; private static final BiFunction BYTES_STRING_ZERO_COPY_CTR = - AndroidSupport.IS_ANDROID ? null : getBytesStringZeroCopyCtr(); + jdkInternalFieldAccess() ? getBytesStringZeroCopyCtr() : null; private static final Function LATIN_BYTES_STRING_ZERO_COPY_CTR = - AndroidSupport.IS_ANDROID ? null : getLatinBytesStringZeroCopyCtr(); + jdkInternalFieldAccess() ? getLatinBytesStringZeroCopyCtr() : null; public static String newCharsStringZeroCopy(char[] data) { - if (AndroidSupport.IS_ANDROID) { + if (!jdkInternalFieldAccess()) { return newCharsStringSlow(data); } if (!STRING_VALUE_FIELD_IS_CHARS) { @@ -944,7 +912,7 @@ private static String newCharsStringSlow(char[] data) { // coder param first to make inline call args // `(buffer.readByte(), buffer.readBytesWithSizeEmbedded())` work. public static String newBytesStringZeroCopy(byte coder, byte[] data) { - if (AndroidSupport.IS_ANDROID) { + if (!jdkInternalFieldAccess()) { return newBytesStringSlow(coder, data); } if (coder == LATIN1) { diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/CollectionSerializers.java b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/CollectionSerializers.java index c2d103eb32..853c9071f9 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/CollectionSerializers.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/CollectionSerializers.java @@ -57,9 +57,10 @@ import org.apache.fory.exception.DeserializationException; import org.apache.fory.exception.ForyException; import org.apache.fory.memory.MemoryBuffer; -import org.apache.fory.platform.AndroidSupport; +import org.apache.fory.memory.MemoryUtils; import org.apache.fory.platform.GraalvmSupport; import org.apache.fory.platform.UnsafeOps; +import org.apache.fory.reflect.FieldAccessor; import org.apache.fory.reflect.ReflectionUtils; import org.apache.fory.resolver.ClassResolver; import org.apache.fory.resolver.TypeInfo; @@ -129,14 +130,13 @@ public ArrayList newCollection(ReadContext readContext) { } public static final class ArraysAsListSerializer extends CollectionSerializer> { - private static final class ArrayFieldOffset { - // Make offset compatible with graalvm native image. - private static final long VALUE; + private static final class ArrayAccess { + private static final FieldAccessor ACCESSOR; static { try { Field arrayField = Class.forName("java.util.Arrays$ArrayList").getDeclaredField("a"); - VALUE = UnsafeOps.objectFieldOffset(arrayField); + ACCESSOR = FieldAccessor.createAccessor(arrayField); } catch (final Exception e) { throw new RuntimeException(e); } @@ -162,9 +162,9 @@ public void write(WriteContext writeContext, List value) { super.write(writeContext, value); } else { Object[] array = - AndroidSupport.IS_ANDROID || GraalvmSupport.IN_GRAALVM_NATIVE_IMAGE + !MemoryUtils.JDK_INTERNAL_FIELD_ACCESS || GraalvmSupport.IN_GRAALVM_NATIVE_IMAGE ? value.toArray() - : (Object[]) UnsafeOps.getObject(value, ArrayFieldOffset.VALUE); + : (Object[]) ArrayAccess.ACCESSOR.getObject(value); writeContext.writeRef(array); } } @@ -553,7 +553,7 @@ public static final class SetFromMapSerializer extends CollectionSerializer(); private static final class JvmSetFromMapAccess { - private static final long MAP_FIELD_OFFSET; + private static final FieldAccessor MAP_ACCESSOR; private static final MethodHandle M_SETTER; private static final MethodHandle S_SETTER; @@ -561,7 +561,7 @@ private static final class JvmSetFromMapAccess { try { Class type = Class.forName("java.util.Collections$SetFromMap"); Field mapField = type.getDeclaredField("m"); - MAP_FIELD_OFFSET = UnsafeOps.objectFieldOffset(mapField); + MAP_ACCESSOR = FieldAccessor.createAccessor(mapField); MethodHandles.Lookup lookup = _JDKAccess._trustedLookup(type); M_SETTER = lookup.findSetter(type, "m", Map.class); S_SETTER = lookup.findSetter(type, "s", Set.class); @@ -588,7 +588,7 @@ public Collection newCollection(ReadContext readContext) { set = Collections.newSetFromMap(mapSerializer.newMap(readContext)); setNumElements(mapSerializer.getAndClearNumElements()); } else { - if (AndroidSupport.IS_ANDROID || GraalvmSupport.IN_GRAALVM_NATIVE_IMAGE) { + if (!MemoryUtils.JDK_INTERNAL_FIELD_ACCESS || GraalvmSupport.IN_GRAALVM_NATIVE_IMAGE) { throw new UnsupportedOperationException( "This runtime cannot read legacy SetFromMap payloads that require hidden JDK field " + "restoration"); @@ -610,12 +610,11 @@ public Collection newCollection(ReadContext readContext) { @Override public Collection newCollection(CopyContext copyContext, Collection originCollection) { assert !config.isXlang(); - if (AndroidSupport.IS_ANDROID || GraalvmSupport.IN_GRAALVM_NATIVE_IMAGE) { + if (!MemoryUtils.JDK_INTERNAL_FIELD_ACCESS || GraalvmSupport.IN_GRAALVM_NATIVE_IMAGE) { return Collections.newSetFromMap(new HashMap(originCollection.size())); } Map map = - (Map) - UnsafeOps.getObject(originCollection, JvmSetFromMapAccess.MAP_FIELD_OFFSET); + (Map) JvmSetFromMapAccess.MAP_ACCESSOR.getObject(originCollection); MapLikeSerializer mapSerializer = (MapLikeSerializer) typeResolver.getSerializer(map.getClass()); Map newMap = mapSerializer.newMap(copyContext, map); @@ -627,7 +626,7 @@ public Collection onCollectionWrite(WriteContext writeContext, Set value) { MemoryBuffer buffer = writeContext.getBuffer(); final Map map; final TypeInfo typeInfo; - if (AndroidSupport.IS_ANDROID || GraalvmSupport.IN_GRAALVM_NATIVE_IMAGE) { + if (!MemoryUtils.JDK_INTERNAL_FIELD_ACCESS || GraalvmSupport.IN_GRAALVM_NATIVE_IMAGE) { HashMap source = new HashMap<>(value.size()); for (Object element : value) { source.put(element, Boolean.TRUE); @@ -635,7 +634,7 @@ public Collection onCollectionWrite(WriteContext writeContext, Set value) { map = source; typeInfo = typeResolver.getTypeInfo(HashMap.class); } else { - map = (Map) UnsafeOps.getObject(value, JvmSetFromMapAccess.MAP_FIELD_OFFSET); + map = (Map) JvmSetFromMapAccess.MAP_ACCESSOR.getObject(value); typeInfo = typeResolver.getTypeInfo(map.getClass()); } MapLikeSerializer mapSerializer = (MapLikeSerializer) typeInfo.getSerializer(); @@ -861,13 +860,13 @@ public static class ArrayBlockingQueueSerializer // Use reflection to get the items array length which represents the capacity. // This avoids race conditions when reading remainingCapacity() and size() separately. - private static final class ItemsOffset { - private static final long VALUE; + private static final class ItemsAccess { + private static final FieldAccessor ACCESSOR; static { try { Field itemsField = ArrayBlockingQueue.class.getDeclaredField("items"); - VALUE = UnsafeOps.objectFieldOffset(itemsField); + ACCESSOR = FieldAccessor.createAccessor(itemsField); } catch (NoSuchFieldException e) { throw new RuntimeException(e); } @@ -879,10 +878,10 @@ public ArrayBlockingQueueSerializer(TypeResolver typeResolver, Class { // Use reflection to get the capacity field directly. // This avoids race conditions when reading remainingCapacity() and size() separately. - private static final class CapacityOffset { - private static final long VALUE; + private static final class CapacityAccess { + private static final FieldAccessor ACCESSOR; static { try { Field capacityField = LinkedBlockingQueue.class.getDeclaredField("capacity"); - VALUE = UnsafeOps.objectFieldOffset(capacityField); + ACCESSOR = FieldAccessor.createAccessor(capacityField); } catch (NoSuchFieldException e) { throw new RuntimeException(e); } @@ -946,10 +945,10 @@ public LinkedBlockingQueueSerializer( } private static int getCapacity(LinkedBlockingQueue queue) { - if (AndroidSupport.IS_ANDROID || GraalvmSupport.IN_GRAALVM_NATIVE_IMAGE) { + if (!MemoryUtils.JDK_INTERNAL_FIELD_ACCESS || GraalvmSupport.IN_GRAALVM_NATIVE_IMAGE) { return queue.size() + queue.remainingCapacity(); } - return UnsafeOps.getInt(queue, CapacityOffset.VALUE); + return CapacityAccess.ACCESSOR.getInt(queue); } @Override diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/MapSerializers.java b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/MapSerializers.java index 953314a03d..b94fd6f366 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/MapSerializers.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/MapSerializers.java @@ -39,8 +39,8 @@ import org.apache.fory.context.ReadContext; import org.apache.fory.context.WriteContext; import org.apache.fory.memory.MemoryBuffer; -import org.apache.fory.platform.AndroidSupport; -import org.apache.fory.platform.UnsafeOps; +import org.apache.fory.memory.MemoryUtils; +import org.apache.fory.reflect.FieldAccessor; import org.apache.fory.reflect.ReflectionUtils; import org.apache.fory.resolver.ClassResolver; import org.apache.fory.resolver.TypeInfo; @@ -336,13 +336,12 @@ public static class EnumMapSerializer extends MapSerializer { private static final byte NORMAL_ENUM_MAP = 0; private static final byte JAVA_SERIALIZED_EMPTY_ENUM_MAP = 1; - private static final class KeyTypeFieldOffset { - // Make offset compatible with graalvm native image. - private static final long VALUE; + private static final class KeyTypeAccess { + private static final FieldAccessor ACCESSOR; static { try { - VALUE = UnsafeOps.objectFieldOffset(EnumMap.class.getDeclaredField("keyType")); + ACCESSOR = FieldAccessor.createAccessor(EnumMap.class.getDeclaredField("keyType")); } catch (final Exception e) { throw new RuntimeException(e); } @@ -360,7 +359,7 @@ public EnumMapSerializer(TypeResolver typeResolver) { @Override public Map onMapWrite(WriteContext writeContext, EnumMap value) { MemoryBuffer buffer = writeContext.getBuffer(); - if (AndroidSupport.IS_ANDROID && value.isEmpty()) { + if (!MemoryUtils.JDK_INTERNAL_FIELD_ACCESS && value.isEmpty()) { buffer.writeByte(JAVA_SERIALIZED_EMPTY_ENUM_MAP); getJavaSerializer().write(writeContext, value); return value; @@ -402,7 +401,7 @@ private static Class getKeyType(EnumMap value) { Enum key = (Enum) value.keySet().iterator().next(); return key.getDeclaringClass(); } - return (Class) UnsafeOps.getObject(value, KeyTypeFieldOffset.VALUE); + return (Class) KeyTypeAccess.ACCESSOR.getObject(value); } private JavaSerializer getJavaSerializer() { diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/SubListSerializers.java b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/SubListSerializers.java index 6aa9158e70..fab1967e4c 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/SubListSerializers.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/SubListSerializers.java @@ -31,8 +31,8 @@ import org.apache.fory.logging.Logger; import org.apache.fory.logging.LoggerFactory; import org.apache.fory.memory.MemoryBuffer; -import org.apache.fory.platform.AndroidSupport; -import org.apache.fory.platform.UnsafeOps; +import org.apache.fory.memory.MemoryUtils; +import org.apache.fory.reflect.FieldAccessor; import org.apache.fory.resolver.TypeResolver; @SuppressWarnings({"rawtypes", "unchecked"}) @@ -210,18 +210,19 @@ private ViewPayload(List source, int offset, int size) { } private static final class ViewFields { - private final long sourceOffset; - private final long offsetOffset; - private final long sizeOffset; + private final FieldAccessor sourceAccessor; + private final FieldAccessor offsetAccessor; + private final FieldAccessor sizeAccessor; - private ViewFields(long sourceOffset, long offsetOffset, long sizeOffset) { - this.sourceOffset = sourceOffset; - this.offsetOffset = offsetOffset; - this.sizeOffset = sizeOffset; + private ViewFields( + FieldAccessor sourceAccessor, FieldAccessor offsetAccessor, FieldAccessor sizeAccessor) { + this.sourceAccessor = sourceAccessor; + this.offsetAccessor = offsetAccessor; + this.sizeAccessor = sizeAccessor; } private static ViewFields create(Class type) { - if (AndroidSupport.IS_ANDROID || Stub.class.isAssignableFrom(type)) { + if (!MemoryUtils.JDK_INTERNAL_FIELD_ACCESS || Stub.class.isAssignableFrom(type)) { return null; } Class cls = type; @@ -282,18 +283,18 @@ private static ViewFields createFromDeclaredFields(Class type) { return null; } return new ViewFields( - UnsafeOps.objectFieldOffset(sourceField), - UnsafeOps.objectFieldOffset(offsetField), - UnsafeOps.objectFieldOffset(sizeField)); + FieldAccessor.createAccessor(sourceField), + FieldAccessor.createAccessor(offsetField), + FieldAccessor.createAccessor(sizeField)); } private ViewPayload get(Collection value) { - List source = (List) UnsafeOps.getObject(value, sourceOffset); + List source = (List) sourceAccessor.getObject(value); if (source == null) { return null; } - int offset = UnsafeOps.getInt(value, offsetOffset); - int size = UnsafeOps.getInt(value, sizeOffset); + int offset = offsetAccessor.getInt(value); + int size = sizeAccessor.getInt(value); return new ViewPayload(source, offset, size); } } diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/SynchronizedSerializers.java b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/SynchronizedSerializers.java index bf2b4b486e..b7f03987f7 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/SynchronizedSerializers.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/SynchronizedSerializers.java @@ -40,8 +40,8 @@ import org.apache.fory.context.WriteContext; import org.apache.fory.logging.Logger; import org.apache.fory.logging.LoggerFactory; -import org.apache.fory.platform.AndroidSupport; -import org.apache.fory.platform.UnsafeOps; +import org.apache.fory.memory.MemoryUtils; +import org.apache.fory.reflect.FieldAccessor; import org.apache.fory.resolver.TypeInfo; import org.apache.fory.resolver.TypeResolver; import org.apache.fory.serializer.Serializer; @@ -53,25 +53,23 @@ public class SynchronizedSerializers { private static final Logger LOG = LoggerFactory.getLogger(SynchronizedSerializers.class); - private static class Offset { - // Graalvm unsafe offset substitution support: Make the call followed by a field store - // directly or by a sign extend node followed directly by a field store. - private static final long SOURCE_COLLECTION_FIELD_OFFSET; - private static final long SOURCE_MAP_FIELD_OFFSET; + private static class SourceAccessors { + private static final FieldAccessor SOURCE_COLLECTION_ACCESSOR; + private static final FieldAccessor SOURCE_MAP_ACCESSOR; static { String clsName = "java.util.Collections$SynchronizedCollection"; try { - SOURCE_COLLECTION_FIELD_OFFSET = - UnsafeOps.UNSAFE.objectFieldOffset(Class.forName(clsName).getDeclaredField("c")); + SOURCE_COLLECTION_ACCESSOR = + FieldAccessor.createAccessor(Class.forName(clsName).getDeclaredField("c")); } catch (Exception e) { LOG.info("Could not access source collection field in {}", clsName); throw new RuntimeException(e); } clsName = "java.util.Collections$SynchronizedMap"; try { - SOURCE_MAP_FIELD_OFFSET = - UnsafeOps.UNSAFE.objectFieldOffset(Class.forName(clsName).getDeclaredField("m")); + SOURCE_MAP_ACCESSOR = + FieldAccessor.createAccessor(Class.forName(clsName).getDeclaredField("m")); } catch (Exception e) { LOG.info("Could not access source map field in {}", clsName); throw new RuntimeException(e); @@ -83,19 +81,19 @@ public static final class SynchronizedCollectionSerializer extends CollectionSerializer { private final Function factory; - private final long offset; + private final FieldAccessor sourceAccessor; public SynchronizedCollectionSerializer( - TypeResolver typeResolver, Class cls, Function factory, long offset) { + TypeResolver typeResolver, Class cls, Function factory, FieldAccessor sourceAccessor) { super(typeResolver, cls, false); this.factory = factory; - this.offset = offset; + this.sourceAccessor = sourceAccessor; } @Override public void write(WriteContext writeContext, Collection object) { Preconditions.checkArgument(object.getClass() == type); - if (AndroidSupport.IS_ANDROID) { + if (!MemoryUtils.JDK_INTERNAL_FIELD_ACCESS) { synchronized (object) { Collection source; if (object instanceof SortedSet) { @@ -111,7 +109,7 @@ public void write(WriteContext writeContext, Collection object) { return; } // the ordinal could be replaced by s.th. else (e.g. a explicitly managed "id") - Object unwrapped = UnsafeOps.getObject(object, offset); + Object unwrapped = sourceAccessor.getObject(object); synchronized (object) { writeContext.writeRef(unwrapped); } @@ -125,7 +123,7 @@ public Collection read(ReadContext readContext) { @Override public Collection copy(CopyContext copyContext, Collection object) { - if (AndroidSupport.IS_ANDROID) { + if (!MemoryUtils.JDK_INTERNAL_FIELD_ACCESS) { synchronized (object) { Collection mutableSource; if (object instanceof SortedSet) { @@ -142,26 +140,26 @@ public Collection copy(CopyContext copyContext, Collection object) { return result; } } - final Object collection = UnsafeOps.getObject(object, offset); + final Object collection = sourceAccessor.getObject(object); return (Collection) factory.apply(copyContext.copyObject(collection)); } } public static final class SynchronizedMapSerializer extends MapSerializer { private final Function factory; - private final long offset; + private final FieldAccessor sourceAccessor; public SynchronizedMapSerializer( - TypeResolver typeResolver, Class cls, Function factory, long offset) { + TypeResolver typeResolver, Class cls, Function factory, FieldAccessor sourceAccessor) { super(typeResolver, cls, false); this.factory = factory; - this.offset = offset; + this.sourceAccessor = sourceAccessor; } @Override public void write(WriteContext writeContext, Map object) { Preconditions.checkArgument(object.getClass() == type); - if (AndroidSupport.IS_ANDROID) { + if (!MemoryUtils.JDK_INTERNAL_FIELD_ACCESS) { synchronized (object) { Map source; if (object instanceof SortedMap) { @@ -175,7 +173,7 @@ public void write(WriteContext writeContext, Map object) { return; } // the ordinal could be replaced by s.th. else (e.g. a explicitly managed "id") - Object unwrapped = UnsafeOps.getObject(object, offset); + Object unwrapped = sourceAccessor.getObject(object); synchronized (object) { writeContext.writeRef(unwrapped); } @@ -183,7 +181,7 @@ public void write(WriteContext writeContext, Map object) { @Override public Map copy(CopyContext copyContext, Map originMap) { - if (AndroidSupport.IS_ANDROID) { + if (!MemoryUtils.JDK_INTERNAL_FIELD_ACCESS) { synchronized (originMap) { Map mutableSource; if (originMap instanceof SortedMap) { @@ -198,7 +196,7 @@ public Map copy(CopyContext copyContext, Map originMap) { return result; } } - final Object unwrappedMap = UnsafeOps.getObject(originMap, offset); + final Object unwrappedMap = sourceAccessor.getObject(originMap); return (Map) factory.apply(copyContext.copyObject(unwrappedMap)); } @@ -225,13 +223,15 @@ private static Serializer createSerializer( typeResolver, factory.f0, factory.f1, - AndroidSupport.IS_ANDROID ? -1 : Offset.SOURCE_COLLECTION_FIELD_OFFSET); + MemoryUtils.JDK_INTERNAL_FIELD_ACCESS + ? SourceAccessors.SOURCE_COLLECTION_ACCESSOR + : null); } else { return new SynchronizedMapSerializer( typeResolver, factory.f0, factory.f1, - AndroidSupport.IS_ANDROID ? -1 : Offset.SOURCE_MAP_FIELD_OFFSET); + MemoryUtils.JDK_INTERNAL_FIELD_ACCESS ? SourceAccessors.SOURCE_MAP_ACCESSOR : null); } } diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/UnmodifiableSerializers.java b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/UnmodifiableSerializers.java index 0bf27e9133..4b0cd0372c 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/UnmodifiableSerializers.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/UnmodifiableSerializers.java @@ -39,8 +39,8 @@ import org.apache.fory.context.WriteContext; import org.apache.fory.logging.Logger; import org.apache.fory.logging.LoggerFactory; -import org.apache.fory.platform.AndroidSupport; -import org.apache.fory.platform.UnsafeOps; +import org.apache.fory.memory.MemoryUtils; +import org.apache.fory.reflect.FieldAccessor; import org.apache.fory.resolver.TypeInfo; import org.apache.fory.resolver.TypeResolver; import org.apache.fory.serializer.Serializer; @@ -52,18 +52,16 @@ public class UnmodifiableSerializers { private static final Logger LOG = LoggerFactory.getLogger(UnmodifiableSerializers.class); - private static class Offset { - // Graalvm unsafe offset substitution support: Make the call followed by a field store - // directly or by a sign extend node followed directly by a field store. - private static final long SOURCE_COLLECTION_FIELD_OFFSET; - private static final long SOURCE_MAP_FIELD_OFFSET; + private static class SourceAccessors { + private static final FieldAccessor SOURCE_COLLECTION_ACCESSOR; + private static final FieldAccessor SOURCE_MAP_ACCESSOR; static { // UnmodifiableList/Set/Etc.. extends UnmodifiableCollection String clsName = "java.util.Collections$UnmodifiableCollection"; try { - SOURCE_COLLECTION_FIELD_OFFSET = - UnsafeOps.UNSAFE.objectFieldOffset(Class.forName(clsName).getDeclaredField("c")); + SOURCE_COLLECTION_ACCESSOR = + FieldAccessor.createAccessor(Class.forName(clsName).getDeclaredField("c")); } catch (Exception e) { LOG.info("Could not access source collection field in {}", clsName); throw new RuntimeException(e); @@ -71,8 +69,8 @@ private static class Offset { clsName = "java.util.Collections$UnmodifiableMap"; try { // UnmodifiableSortedMap/UnmodifiableNavigableMap extends UnmodifiableMap - SOURCE_MAP_FIELD_OFFSET = - UnsafeOps.UNSAFE.objectFieldOffset(Class.forName(clsName).getDeclaredField("m")); + SOURCE_MAP_ACCESSOR = + FieldAccessor.createAccessor(Class.forName(clsName).getDeclaredField("m")); } catch (Exception e) { LOG.info("Could not access source map field in {}", clsName); throw new RuntimeException(e); @@ -83,19 +81,19 @@ private static class Offset { public static final class UnmodifiableCollectionSerializer extends CollectionSerializer { private final Function factory; - private final long offset; + private final FieldAccessor sourceAccessor; public UnmodifiableCollectionSerializer( - TypeResolver typeResolver, Class cls, Function factory, long offset) { + TypeResolver typeResolver, Class cls, Function factory, FieldAccessor sourceAccessor) { super(typeResolver, cls, false); this.factory = factory; - this.offset = offset; + this.sourceAccessor = sourceAccessor; } @Override public void write(WriteContext writeContext, Collection value) { Preconditions.checkArgument(value.getClass() == type); - if (AndroidSupport.IS_ANDROID) { + if (!MemoryUtils.JDK_INTERNAL_FIELD_ACCESS) { Collection source; if (value instanceof SortedSet) { source = new TreeSet(((SortedSet) value).comparator()); @@ -108,7 +106,7 @@ public void write(WriteContext writeContext, Collection value) { writeContext.writeRef(source, sourceCollectionTypeInfo(typeResolver, type)); return; } - writeContext.writeRef(UnsafeOps.getObject(value, offset)); + writeContext.writeRef(sourceAccessor.getObject(value)); } @Override @@ -118,7 +116,7 @@ public Collection read(ReadContext readContext) { @Override public Collection copy(CopyContext copyContext, Collection object) { - if (AndroidSupport.IS_ANDROID) { + if (!MemoryUtils.JDK_INTERNAL_FIELD_ACCESS) { Collection mutableSource; if (object instanceof SortedSet) { Object comparator = copyContext.copyObject(((SortedSet) object).comparator()); @@ -133,26 +131,25 @@ public Collection copy(CopyContext copyContext, Collection object) { copyElements(copyContext, object, mutableSource); return result; } - return (Collection) - factory.apply(copyContext.copyObject(UnsafeOps.getObject(object, offset))); + return (Collection) factory.apply(copyContext.copyObject(sourceAccessor.getObject(object))); } } public static final class UnmodifiableMapSerializer extends MapSerializer { private final Function factory; - private final long offset; + private final FieldAccessor sourceAccessor; public UnmodifiableMapSerializer( - TypeResolver typeResolver, Class cls, Function factory, long offset) { + TypeResolver typeResolver, Class cls, Function factory, FieldAccessor sourceAccessor) { super(typeResolver, cls, false); this.factory = factory; - this.offset = offset; + this.sourceAccessor = sourceAccessor; } @Override public void write(WriteContext writeContext, Map value) { Preconditions.checkArgument(value.getClass() == type); - if (AndroidSupport.IS_ANDROID) { + if (!MemoryUtils.JDK_INTERNAL_FIELD_ACCESS) { Map source; if (value instanceof SortedMap) { source = new TreeMap(((SortedMap) value).comparator()); @@ -163,12 +160,12 @@ public void write(WriteContext writeContext, Map value) { writeContext.writeRef(source, sourceMapTypeInfo(typeResolver, type)); return; } - writeContext.writeRef(UnsafeOps.getObject(value, offset)); + writeContext.writeRef(sourceAccessor.getObject(value)); } @Override public Map copy(CopyContext copyContext, Map originMap) { - if (AndroidSupport.IS_ANDROID) { + if (!MemoryUtils.JDK_INTERNAL_FIELD_ACCESS) { Map mutableSource; if (originMap instanceof SortedMap) { Object comparator = copyContext.copyObject(((SortedMap) originMap).comparator()); @@ -181,7 +178,7 @@ public Map copy(CopyContext copyContext, Map originMap) { copyEntry(copyContext, originMap, mutableSource); return result; } - return (Map) factory.apply(copyContext.copyObject(UnsafeOps.getObject(originMap, offset))); + return (Map) factory.apply(copyContext.copyObject(sourceAccessor.getObject(originMap))); } @Override @@ -206,13 +203,15 @@ private static Serializer createSerializer( typeResolver, factory.f0, factory.f1, - AndroidSupport.IS_ANDROID ? -1 : Offset.SOURCE_COLLECTION_FIELD_OFFSET); + MemoryUtils.JDK_INTERNAL_FIELD_ACCESS + ? SourceAccessors.SOURCE_COLLECTION_ACCESSOR + : null); } else { return new UnmodifiableMapSerializer( typeResolver, factory.f0, factory.f1, - AndroidSupport.IS_ANDROID ? -1 : Offset.SOURCE_MAP_FIELD_OFFSET); + MemoryUtils.JDK_INTERNAL_FIELD_ACCESS ? SourceAccessors.SOURCE_MAP_ACCESSOR : null); } } diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/scala/SingletonCollectionSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/scala/SingletonCollectionSerializer.java index 9d96e02274..e17e309a02 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/scala/SingletonCollectionSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/scala/SingletonCollectionSerializer.java @@ -26,7 +26,7 @@ import org.apache.fory.exception.ForyException; import org.apache.fory.platform.AndroidSupport; import org.apache.fory.platform.GraalvmSupport; -import org.apache.fory.platform.UnsafeOps; +import org.apache.fory.reflect.FieldAccessor; import org.apache.fory.resolver.TypeResolver; import org.apache.fory.serializer.collection.CollectionLikeSerializer; import org.apache.fory.util.Preconditions; @@ -39,8 +39,7 @@ @SuppressWarnings("rawtypes") public class SingletonCollectionSerializer extends CollectionLikeSerializer { private final Field field; - private Object base = null; - private long offset = -1; + private FieldAccessor accessor; public SingletonCollectionSerializer(TypeResolver typeResolver, Class cls) { super(typeResolver, cls, false); @@ -78,13 +77,12 @@ public Object read(ReadContext readContext) { throw new ForyException("Failed to read Scala singleton field: " + type, e); } } - long offset = this.offset; - if (offset == -1) { + FieldAccessor accessor = this.accessor; + if (accessor == null) { Preconditions.checkArgument(!GraalvmSupport.isGraalBuildTime()); - offset = this.offset = UnsafeOps.UNSAFE.staticFieldOffset(field); - base = UnsafeOps.UNSAFE.staticFieldBase(field); + accessor = this.accessor = FieldAccessor.createStaticAccessor(field); } - return UnsafeOps.getObject(base, offset); + return accessor.getObject(null); } @Override diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/scala/SingletonMapSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/scala/SingletonMapSerializer.java index f409d811b9..9ef08ac09b 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/scala/SingletonMapSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/scala/SingletonMapSerializer.java @@ -26,7 +26,7 @@ import org.apache.fory.exception.ForyException; import org.apache.fory.platform.AndroidSupport; import org.apache.fory.platform.GraalvmSupport; -import org.apache.fory.platform.UnsafeOps; +import org.apache.fory.reflect.FieldAccessor; import org.apache.fory.resolver.TypeResolver; import org.apache.fory.serializer.collection.MapLikeSerializer; import org.apache.fory.util.Preconditions; @@ -39,8 +39,7 @@ @SuppressWarnings("rawtypes") public class SingletonMapSerializer extends MapLikeSerializer { private final Field field; - private Object base = null; - private long offset = -1; + private FieldAccessor accessor; public SingletonMapSerializer(TypeResolver typeResolver, Class cls) { super(typeResolver, cls, false); @@ -78,13 +77,12 @@ public Object read(ReadContext readContext) { throw new ForyException("Failed to read Scala singleton field: " + type, e); } } - long offset = this.offset; - if (offset == -1) { + FieldAccessor accessor = this.accessor; + if (accessor == null) { Preconditions.checkArgument(!GraalvmSupport.isGraalBuildTime()); - offset = this.offset = UnsafeOps.UNSAFE.staticFieldOffset(field); - base = UnsafeOps.UNSAFE.staticFieldBase(field); + accessor = this.accessor = FieldAccessor.createStaticAccessor(field); } - return UnsafeOps.getObject(base, offset); + return accessor.getObject(null); } @Override diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/scala/SingletonObjectSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/scala/SingletonObjectSerializer.java index acd991ab93..408da2ebc3 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/scala/SingletonObjectSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/scala/SingletonObjectSerializer.java @@ -25,7 +25,7 @@ import org.apache.fory.exception.ForyException; import org.apache.fory.platform.AndroidSupport; import org.apache.fory.platform.GraalvmSupport; -import org.apache.fory.platform.UnsafeOps; +import org.apache.fory.reflect.FieldAccessor; import org.apache.fory.resolver.TypeResolver; import org.apache.fory.serializer.Serializer; import org.apache.fory.util.Preconditions; @@ -37,8 +37,7 @@ @SuppressWarnings("rawtypes") public class SingletonObjectSerializer extends Serializer { private final Field field; - private Object base = null; - private long offset = -1; + private FieldAccessor accessor; public SingletonObjectSerializer(TypeResolver typeResolver, Class type) { super(typeResolver.getConfig(), type); @@ -71,12 +70,11 @@ public Object read(ReadContext readContext) { throw new ForyException("Failed to read Scala singleton field: " + type, e); } } - long offset = this.offset; - if (offset == -1) { + FieldAccessor accessor = this.accessor; + if (accessor == null) { Preconditions.checkArgument(!GraalvmSupport.isGraalBuildTime()); - offset = this.offset = UnsafeOps.UNSAFE.staticFieldOffset(field); - base = UnsafeOps.UNSAFE.staticFieldBase(field); + accessor = this.accessor = FieldAccessor.createStaticAccessor(field); } - return UnsafeOps.getObject(base, offset); + return accessor.getObject(null); } } diff --git a/java/fory-core/src/main/java/org/apache/fory/util/unsafe/_JDKAccess.java b/java/fory-core/src/main/java/org/apache/fory/util/unsafe/_JDKAccess.java index bd2f143666..4787acc98d 100644 --- a/java/fory-core/src/main/java/org/apache/fory/util/unsafe/_JDKAccess.java +++ b/java/fory-core/src/main/java/org/apache/fory/util/unsafe/_JDKAccess.java @@ -59,7 +59,10 @@ public class _JDKAccess { // CHECKSTYLE.ON:TypeName public static final boolean IS_OPEN_J9; public static final Unsafe UNSAFE; - public static final boolean BYTE_ARRAY_STREAM_WRAP_SUPPORTED; + // Root classes use Unsafe for JDK internal fields. A JDK25 multi-release _JDKAccess must keep + // this API surface and implement supported cases with VarHandle, or set this false so callers + // choose public fallbacks. + public static final boolean JDK_INTERNAL_FIELD_ACCESS; public static final Class _INNER_UNSAFE_CLASS; public static final Object _INNER_UNSAFE; @@ -75,7 +78,7 @@ public class _JDKAccess { throw new UnsupportedOperationException("Unsafe is not supported in this platform."); } UNSAFE = unsafe; - BYTE_ARRAY_STREAM_WRAP_SUPPORTED = true; + JDK_INTERNAL_FIELD_ACCESS = true; if (JdkVersion.MAJOR_VERSION >= 11) { try { Field theInternalUnsafeField = Unsafe.class.getDeclaredField("theInternalUnsafe"); @@ -93,6 +96,76 @@ public class _JDKAccess { private static final ClassValueCache lookupCache = ClassValueCache.newClassKeyCache(32); + public static final boolean STRING_VALUE_FIELD_IS_CHARS; + public static final boolean STRING_VALUE_FIELD_IS_BYTES; + public static final boolean STRING_HAS_COUNT_OFFSET; + private static final long STRING_VALUE_FIELD_OFFSET; + private static final long STRING_COUNT_FIELD_OFFSET; + private static final long STRING_OFFSET_FIELD_OFFSET; + + static { + try { + Field valueField = String.class.getDeclaredField("value"); + STRING_VALUE_FIELD_IS_CHARS = valueField.getType() == char[].class; + STRING_VALUE_FIELD_IS_BYTES = valueField.getType() == byte[].class; + STRING_VALUE_FIELD_OFFSET = UNSAFE.objectFieldOffset(valueField); + Field countField = getStringFieldNullable("count"); + Field offsetField = getStringFieldNullable("offset"); + if (countField != null || offsetField != null) { + Preconditions.checkArgument( + countField != null && offsetField != null, "Current jdk not supported"); + Preconditions.checkArgument( + countField.getType() == int.class && offsetField.getType() == int.class, + "Current jdk not supported"); + STRING_HAS_COUNT_OFFSET = true; + STRING_COUNT_FIELD_OFFSET = UNSAFE.objectFieldOffset(countField); + STRING_OFFSET_FIELD_OFFSET = UNSAFE.objectFieldOffset(offsetField); + } else { + STRING_HAS_COUNT_OFFSET = false; + STRING_COUNT_FIELD_OFFSET = -1; + STRING_OFFSET_FIELD_OFFSET = -1; + } + } catch (NoSuchFieldException e) { + throw new RuntimeException(e); + } + } + + private static Field getStringFieldNullable(String fieldName) { + try { + return String.class.getDeclaredField(fieldName); + } catch (NoSuchFieldException e) { + return null; + } + } + + private static class StringCoderField { + private static final long OFFSET; + + static { + try { + OFFSET = UNSAFE.objectFieldOffset(String.class.getDeclaredField("coder")); + } catch (NoSuchFieldException e) { + throw new RuntimeException(e); + } + } + } + + public static Object getStringValue(String value) { + return UNSAFE.getObject(value, STRING_VALUE_FIELD_OFFSET); + } + + public static byte getStringCoder(String value) { + return UNSAFE.getByte(value, StringCoderField.OFFSET); + } + + public static int getStringOffset(String value) { + return UNSAFE.getInt(value, STRING_OFFSET_FIELD_OFFSET); + } + + public static int getStringCount(String value) { + return UNSAFE.getInt(value, STRING_COUNT_FIELD_OFFSET); + } + // CHECKSTYLE.OFF:MethodName public static Lookup _trustedLookup(Class objectClass) { diff --git a/java/fory-core/src/test/java/org/apache/fory/codegen/JaninoUtilsTest.java b/java/fory-core/src/test/java/org/apache/fory/codegen/JaninoUtilsTest.java index 901a22e0ca..1189ce7631 100644 --- a/java/fory-core/src/test/java/org/apache/fory/codegen/JaninoUtilsTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/codegen/JaninoUtilsTest.java @@ -25,7 +25,6 @@ import java.util.List; import java.util.Map; import java.util.function.Function; -import org.apache.fory.platform.UnsafeOps; import org.apache.fory.reflect.ReflectionUtils; import org.apache.fory.test.bean.Struct; import org.apache.fory.util.ClassLoaderUtils; @@ -254,12 +253,8 @@ public WeakReference> janinoCompileDependentClass(Class de // Compile all sources compiler.compile(sourceFinder.resources().toArray(new Resource[0])); // See https://github.com/janino-compiler/janino/issues/173 - UnsafeOps.putObject( - classLoader, ReflectionUtils.getFieldOffset(classLoader.getClass(), "classLoader"), null); - UnsafeOps.putObject( - classLoader, - ReflectionUtils.getFieldOffset(classLoader.getClass(), "loadedIClasses"), - null); + ReflectionUtils.setObjectFieldValue(classLoader, "classLoader", null); + ReflectionUtils.setObjectFieldValue(classLoader, "loadedIClasses", null); byte[] byteCodes = classes.entrySet().iterator().next().getValue(); ClassLoaderUtils.tryDefineClassesInClassLoader("B", dep, dep.getClassLoader(), byteCodes); Class cls = dep.getClassLoader().loadClass("B"); diff --git a/java/fory-core/src/test/java/org/apache/fory/memory/MemoryBufferTest.java b/java/fory-core/src/test/java/org/apache/fory/memory/MemoryBufferTest.java index 9ee62e3876..03f38a9a16 100644 --- a/java/fory-core/src/test/java/org/apache/fory/memory/MemoryBufferTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/memory/MemoryBufferTest.java @@ -84,7 +84,7 @@ public void testBufferWrite() { @Test public void testByteArrayStreamWrap() { - if (!MemoryUtils.BYTE_ARRAY_STREAM_WRAP_SUPPORTED) { + if (!MemoryUtils.JDK_INTERNAL_FIELD_ACCESS) { return; } ByteArrayOutputStream outputStream = new ByteArrayOutputStream(8); diff --git a/java/fory-core/src/test/java/org/apache/fory/serializer/AndroidDynamicFeatureTest.java b/java/fory-core/src/test/java/org/apache/fory/serializer/AndroidDynamicFeatureTest.java index 5e251c2d06..667de876ae 100644 --- a/java/fory-core/src/test/java/org/apache/fory/serializer/AndroidDynamicFeatureTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/serializer/AndroidDynamicFeatureTest.java @@ -174,8 +174,8 @@ private static void verifyOutputStreamSerialization(Fory fory) { private static void verifyMemoryUtilsStreamWrapGuards() { check( - !MemoryUtils.BYTE_ARRAY_STREAM_WRAP_SUPPORTED, - "Android must report byte-array stream wrapping unsupported"); + !MemoryUtils.JDK_INTERNAL_FIELD_ACCESS, + "Android must report JDK internal field access unsupported"); expectUnsupportedAndroidWrap( () -> MemoryUtils.wrap(new ByteArrayOutputStream(), MemoryUtils.buffer(8)), "ByteArrayOutputStream direct wrapping"); diff --git a/java/fory-core/src/test/java/org/apache/fory/serializer/StringSerializerTest.java b/java/fory-core/src/test/java/org/apache/fory/serializer/StringSerializerTest.java index 95c41e6714..b621677bad 100644 --- a/java/fory-core/src/test/java/org/apache/fory/serializer/StringSerializerTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/serializer/StringSerializerTest.java @@ -36,10 +36,9 @@ import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.memory.MemoryUtils; import org.apache.fory.platform.JdkVersion; -import org.apache.fory.platform.UnsafeOps; -import org.apache.fory.reflect.ReflectionUtils; import org.apache.fory.util.MathUtils; import org.apache.fory.util.StringUtils; +import org.apache.fory.util.unsafe._JDKAccess; import org.testng.Assert; import org.testng.SkipException; import org.testng.annotations.DataProvider; @@ -138,8 +137,7 @@ private static boolean writeJavaStringZeroCopy(MemoryBuffer buffer, String value } static void writeJDK8String(MemoryBuffer buffer, String value) { - final char[] chars = - (char[]) UnsafeOps.getObject(value, ReflectionUtils.getFieldOffset(String.class, "value")); + final char[] chars = (char[]) _JDKAccess.getStringValue(value); int numBytes = MathUtils.doubleExact(value.length()); buffer.writeCharsWithSize(chars); } diff --git a/java/fory-core/src/test/java/org/apache/fory/serializer/collection/SynchronizedSerializersTest.java b/java/fory-core/src/test/java/org/apache/fory/serializer/collection/SynchronizedSerializersTest.java index c12d5661c2..bd1d7fb95a 100644 --- a/java/fory-core/src/test/java/org/apache/fory/serializer/collection/SynchronizedSerializersTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/serializer/collection/SynchronizedSerializersTest.java @@ -39,21 +39,13 @@ import org.apache.fory.ForyTestBase; import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.memory.MemoryUtils; -import org.apache.fory.platform.UnsafeOps; +import org.apache.fory.reflect.FieldAccessor; import org.apache.fory.reflect.ReflectionUtils; import org.apache.fory.serializer.Serializer; import org.apache.fory.test.bean.CollectionFields; import org.testng.annotations.Test; public class SynchronizedSerializersTest extends ForyTestBase { - - static long SOURCE_COLLECTION_FIELD_OFFSET = - ReflectionUtils.getFieldOffset( - Collections.synchronizedCollection(Collections.emptyList()).getClass(), "c"); - static long SOURCE_MAP_FIELD_OFFSET = - ReflectionUtils.getFieldOffset( - Collections.synchronizedMap(Collections.emptyMap()).getClass(), "m"); - @Test public void testWrite() throws Exception { Fory fory = Fory.builder().withXlang(false).requireClassRegistration(false).build(); @@ -77,16 +69,13 @@ public void testWrite() throws Exception { writeSerializer(fory, serializer, buffer, value); Object newObj = readSerializer(fory, serializer, buffer); assertEquals(newObj.getClass(), value.getClass()); - long sourceCollectionFieldOffset = - Collection.class.isAssignableFrom(value.getClass()) - ? SOURCE_COLLECTION_FIELD_OFFSET - : SOURCE_MAP_FIELD_OFFSET; - Object innerValue = UnsafeOps.getObject(value, sourceCollectionFieldOffset); - Object newValue = UnsafeOps.getObject(newObj, sourceCollectionFieldOffset); + FieldAccessor sourceAccessor = sourceAccessor(value.getClass()); + Object innerValue = sourceAccessor.getObject(value); + Object newValue = sourceAccessor.getObject(newObj); assertEquals(innerValue, newValue); newObj = serDe(fory, value); - innerValue = UnsafeOps.getObject(value, sourceCollectionFieldOffset); - newValue = UnsafeOps.getObject(newObj, sourceCollectionFieldOffset); + innerValue = sourceAccessor.getObject(value); + newValue = sourceAccessor.getObject(newObj); assertEquals(innerValue, newValue); assertTrue( fory.getTypeResolver() @@ -96,6 +85,11 @@ public void testWrite() throws Exception { } } + private static FieldAccessor sourceAccessor(Class cls) { + String fieldName = Collection.class.isAssignableFrom(cls) ? "c" : "m"; + return FieldAccessor.createAccessor(ReflectionUtils.getField(cls, fieldName)); + } + @Test(dataProvider = "javaFory") public void testCollectionFieldSerializers(Fory fory) { CollectionFields obj = createCollectionFields(); diff --git a/java/fory-core/src/test/java/org/apache/fory/serializer/collection/UnmodifiableSerializersTest.java b/java/fory-core/src/test/java/org/apache/fory/serializer/collection/UnmodifiableSerializersTest.java index d2fa2ae345..74ea6c9181 100644 --- a/java/fory-core/src/test/java/org/apache/fory/serializer/collection/UnmodifiableSerializersTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/serializer/collection/UnmodifiableSerializersTest.java @@ -45,7 +45,7 @@ import org.apache.fory.ForyTestBase; import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.memory.MemoryUtils; -import org.apache.fory.platform.UnsafeOps; +import org.apache.fory.reflect.FieldAccessor; import org.apache.fory.reflect.ReflectionUtils; import org.apache.fory.serializer.Serializer; import org.apache.fory.test.bean.CollectionFields; @@ -54,13 +54,6 @@ import org.testng.annotations.Test; public class UnmodifiableSerializersTest extends ForyTestBase { - static long SOURCE_COLLECTION_FIELD_OFFSET = - ReflectionUtils.getFieldOffset( - Collections.synchronizedCollection(Collections.emptyList()).getClass(), "c"); - static long SOURCE_MAP_FIELD_OFFSET = - ReflectionUtils.getFieldOffset( - Collections.synchronizedMap(Collections.emptyMap()).getClass(), "m"); - @SuppressWarnings("unchecked") @Test public void testWrite() throws Exception { @@ -84,17 +77,14 @@ public void testWrite() throws Exception { writeSerializer(fory, serializer, buffer, value); Object newObj = readSerializer(fory, serializer, buffer); assertEquals(newObj.getClass(), value.getClass()); - long sourceCollectionFieldOffset = - Collection.class.isAssignableFrom(value.getClass()) - ? SOURCE_COLLECTION_FIELD_OFFSET - : SOURCE_MAP_FIELD_OFFSET; - Object innerValue = UnsafeOps.getObject(value, sourceCollectionFieldOffset); - Object newValue = UnsafeOps.getObject(newObj, sourceCollectionFieldOffset); + FieldAccessor sourceAccessor = sourceAccessor(value.getClass()); + Object innerValue = sourceAccessor.getObject(value); + Object newValue = sourceAccessor.getObject(newObj); assertEquals(innerValue, newValue); newObj = serDe(fory, value); - innerValue = UnsafeOps.getObject(value, sourceCollectionFieldOffset); - newValue = UnsafeOps.getObject(newObj, sourceCollectionFieldOffset); + innerValue = sourceAccessor.getObject(value); + newValue = sourceAccessor.getObject(newObj); assertEquals(innerValue, newValue); assertTrue( fory.getTypeResolver() @@ -104,6 +94,11 @@ public void testWrite() throws Exception { } } + private static FieldAccessor sourceAccessor(Class cls) { + String fieldName = Collection.class.isAssignableFrom(cls) ? "c" : "m"; + return FieldAccessor.createAccessor(ReflectionUtils.getField(cls, fieldName)); + } + public static CollectionFields createCollectionFields() { CollectionFields obj = new CollectionFields(); obj.collection = Collections.unmodifiableCollection(Arrays.asList(1, 2)); From 1187a4a667e1a587cbc42acde7f0ee6005617b93 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sat, 23 May 2026 23:04:08 +0800 Subject: [PATCH 11/69] refactor(java): move JDK access utilities to platform internal --- .../src/main/java/org/apache/fory/memory/MemoryUtils.java | 2 +- .../src/main/java/org/apache/fory/platform/UnsafeOps.java | 2 +- .../{util/unsafe => platform/internal}/DefineClass.java | 2 +- .../fory/{util/unsafe => platform/internal}/_JDKAccess.java | 2 +- .../fory/{util/unsafe => platform/internal}/_Lookup.java | 2 +- .../main/java/org/apache/fory/reflect/FieldAccessor.java | 2 +- .../main/java/org/apache/fory/reflect/ObjectCreators.java | 2 +- .../main/java/org/apache/fory/reflect/ReflectionUtils.java | 2 +- .../org/apache/fory/serializer/ExceptionSerializers.java | 2 +- .../java/org/apache/fory/serializer/LambdaSerializer.java | 2 +- .../org/apache/fory/serializer/ObjectStreamSerializer.java | 2 +- .../apache/fory/serializer/ReplaceResolveSerializer.java | 2 +- .../apache/fory/serializer/SerializedLambdaSerializer.java | 2 +- .../main/java/org/apache/fory/serializer/Serializers.java | 2 +- .../java/org/apache/fory/serializer/StringSerializer.java | 2 +- .../fory/serializer/collection/CollectionSerializers.java | 2 +- .../collection/ImmutableCollectionSerializers.java | 2 +- .../main/java/org/apache/fory/util/ClassLoaderUtils.java | 2 +- .../main/java/org/apache/fory/util/DefaultValueUtils.java | 2 +- .../main/java/org/apache/fory/util/function/Functions.java | 2 +- .../main/java/org/apache/fory/util/record/RecordUtils.java | 2 +- .../org.apache.fory/fory-core/native-image.properties | 6 +++--- java/fory-core/src/test/java/org/apache/fory/TestUtils.java | 2 +- .../org/apache/fory/collection/MultiKeyWeakMapTest.java | 2 +- .../{util/unsafe => platform/internal}/DefineClassTest.java | 2 +- .../{util/unsafe => platform/internal}/JDKAccessTest.java | 2 +- .../org/apache/fory/serializer/StringSerializerTest.java | 2 +- .../fory/extension/serializer/ProtobufSerializer.java | 2 +- .../apache/fory/format/type/CustomTypeEncoderRegistry.java | 2 +- .../org/apache/fory/serializer/scala/RangeSerializer.scala | 2 +- 30 files changed, 32 insertions(+), 32 deletions(-) rename java/fory-core/src/main/java/org/apache/fory/{util/unsafe => platform/internal}/DefineClass.java (98%) rename java/fory-core/src/main/java/org/apache/fory/{util/unsafe => platform/internal}/_JDKAccess.java (99%) rename java/fory-core/src/main/java/org/apache/fory/{util/unsafe => platform/internal}/_Lookup.java (99%) rename java/fory-core/src/test/java/org/apache/fory/{util/unsafe => platform/internal}/DefineClassTest.java (98%) rename java/fory-core/src/test/java/org/apache/fory/{util/unsafe => platform/internal}/JDKAccessTest.java (99%) diff --git a/java/fory-core/src/main/java/org/apache/fory/memory/MemoryUtils.java b/java/fory-core/src/main/java/org/apache/fory/memory/MemoryUtils.java index 9194937c4e..cf626123b1 100644 --- a/java/fory-core/src/main/java/org/apache/fory/memory/MemoryUtils.java +++ b/java/fory-core/src/main/java/org/apache/fory/memory/MemoryUtils.java @@ -23,7 +23,7 @@ import java.io.ByteArrayOutputStream; import java.nio.ByteBuffer; import org.apache.fory.platform.AndroidSupport; -import org.apache.fory.util.unsafe._JDKAccess; +import org.apache.fory.platform.internal._JDKAccess; /** Memory utils for fory. */ public class MemoryUtils { diff --git a/java/fory-core/src/main/java/org/apache/fory/platform/UnsafeOps.java b/java/fory-core/src/main/java/org/apache/fory/platform/UnsafeOps.java index 7ca7d09bd0..8e214946bc 100644 --- a/java/fory-core/src/main/java/org/apache/fory/platform/UnsafeOps.java +++ b/java/fory-core/src/main/java/org/apache/fory/platform/UnsafeOps.java @@ -22,8 +22,8 @@ import java.lang.reflect.Field; import java.lang.reflect.Method; import org.apache.fory.annotation.Internal; +import org.apache.fory.platform.internal._JDKAccess; import org.apache.fory.util.ExceptionUtils; -import org.apache.fory.util.unsafe._JDKAccess; import sun.misc.Unsafe; // Derived from Apache Spark's unsafe memory utility. diff --git a/java/fory-core/src/main/java/org/apache/fory/util/unsafe/DefineClass.java b/java/fory-core/src/main/java/org/apache/fory/platform/internal/DefineClass.java similarity index 98% rename from java/fory-core/src/main/java/org/apache/fory/util/unsafe/DefineClass.java rename to java/fory-core/src/main/java/org/apache/fory/platform/internal/DefineClass.java index eca71ea811..c47ffa9d44 100644 --- a/java/fory-core/src/main/java/org/apache/fory/util/unsafe/DefineClass.java +++ b/java/fory-core/src/main/java/org/apache/fory/platform/internal/DefineClass.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.fory.util.unsafe; +package org.apache.fory.platform.internal; import java.lang.invoke.MethodHandle; import java.lang.invoke.MethodHandles; diff --git a/java/fory-core/src/main/java/org/apache/fory/util/unsafe/_JDKAccess.java b/java/fory-core/src/main/java/org/apache/fory/platform/internal/_JDKAccess.java similarity index 99% rename from java/fory-core/src/main/java/org/apache/fory/util/unsafe/_JDKAccess.java rename to java/fory-core/src/main/java/org/apache/fory/platform/internal/_JDKAccess.java index 4787acc98d..514a8a9d57 100644 --- a/java/fory-core/src/main/java/org/apache/fory/util/unsafe/_JDKAccess.java +++ b/java/fory-core/src/main/java/org/apache/fory/platform/internal/_JDKAccess.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.fory.util.unsafe; +package org.apache.fory.platform.internal; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; diff --git a/java/fory-core/src/main/java/org/apache/fory/util/unsafe/_Lookup.java b/java/fory-core/src/main/java/org/apache/fory/platform/internal/_Lookup.java similarity index 99% rename from java/fory-core/src/main/java/org/apache/fory/util/unsafe/_Lookup.java rename to java/fory-core/src/main/java/org/apache/fory/platform/internal/_Lookup.java index b719ac0afb..2ece8bb9ce 100644 --- a/java/fory-core/src/main/java/org/apache/fory/util/unsafe/_Lookup.java +++ b/java/fory-core/src/main/java/org/apache/fory/platform/internal/_Lookup.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.fory.util.unsafe; +package org.apache.fory.platform.internal; import java.lang.invoke.MethodHandle; import java.lang.invoke.MethodHandles; diff --git a/java/fory-core/src/main/java/org/apache/fory/reflect/FieldAccessor.java b/java/fory-core/src/main/java/org/apache/fory/reflect/FieldAccessor.java index 2ff89fc529..b4919e5da2 100644 --- a/java/fory-core/src/main/java/org/apache/fory/reflect/FieldAccessor.java +++ b/java/fory-core/src/main/java/org/apache/fory/reflect/FieldAccessor.java @@ -38,6 +38,7 @@ import org.apache.fory.platform.AndroidSupport; import org.apache.fory.platform.GraalvmSupport; import org.apache.fory.platform.UnsafeOps; +import org.apache.fory.platform.internal._JDKAccess; import org.apache.fory.type.TypeUtils; import org.apache.fory.util.Preconditions; import org.apache.fory.util.function.Functions; @@ -46,7 +47,6 @@ import org.apache.fory.util.function.ToFloatFunction; import org.apache.fory.util.function.ToShortFunction; import org.apache.fory.util.record.RecordUtils; -import org.apache.fory.util.unsafe._JDKAccess; /** Field accessor for primitive types and object types. */ @SuppressWarnings({"unchecked", "rawtypes"}) diff --git a/java/fory-core/src/main/java/org/apache/fory/reflect/ObjectCreators.java b/java/fory-core/src/main/java/org/apache/fory/reflect/ObjectCreators.java index 00fc668539..065a82b237 100644 --- a/java/fory-core/src/main/java/org/apache/fory/reflect/ObjectCreators.java +++ b/java/fory-core/src/main/java/org/apache/fory/reflect/ObjectCreators.java @@ -30,8 +30,8 @@ import org.apache.fory.platform.GraalvmSupport; import org.apache.fory.platform.JdkVersion; import org.apache.fory.platform.UnsafeOps; +import org.apache.fory.platform.internal._JDKAccess; import org.apache.fory.util.record.RecordUtils; -import org.apache.fory.util.unsafe._JDKAccess; /** * Factory class for creating and caching {@link ObjectCreator} instances. diff --git a/java/fory-core/src/main/java/org/apache/fory/reflect/ReflectionUtils.java b/java/fory-core/src/main/java/org/apache/fory/reflect/ReflectionUtils.java index e09b3361bd..de19e8e262 100644 --- a/java/fory-core/src/main/java/org/apache/fory/reflect/ReflectionUtils.java +++ b/java/fory-core/src/main/java/org/apache/fory/reflect/ReflectionUtils.java @@ -53,11 +53,11 @@ import org.apache.fory.platform.AndroidSupport; import org.apache.fory.platform.GraalvmSupport; import org.apache.fory.platform.UnsafeOps; +import org.apache.fory.platform.internal._JDKAccess; import org.apache.fory.util.ExceptionUtils; import org.apache.fory.util.Preconditions; import org.apache.fory.util.StringUtils; import org.apache.fory.util.function.Functions; -import org.apache.fory.util.unsafe._JDKAccess; /** Reflection util. */ @Internal diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/ExceptionSerializers.java b/java/fory-core/src/main/java/org/apache/fory/serializer/ExceptionSerializers.java index e648da8716..4c32c84e94 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/ExceptionSerializers.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/ExceptionSerializers.java @@ -42,12 +42,12 @@ import org.apache.fory.meta.TypeDef; import org.apache.fory.platform.AndroidSupport; import org.apache.fory.platform.GraalvmSupport; +import org.apache.fory.platform.internal._JDKAccess; import org.apache.fory.reflect.FieldAccessor; import org.apache.fory.reflect.ObjectCreator; import org.apache.fory.reflect.ObjectCreators; import org.apache.fory.reflect.ReflectionUtils; import org.apache.fory.resolver.TypeResolver; -import org.apache.fory.util.unsafe._JDKAccess; /** Serializers for {@link Throwable} and {@link StackTraceElement}. */ @SuppressWarnings({"rawtypes", "unchecked"}) diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/LambdaSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/LambdaSerializer.java index f7f68a1a6e..6376b62d03 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/LambdaSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/LambdaSerializer.java @@ -28,10 +28,10 @@ import org.apache.fory.context.ReadContext; import org.apache.fory.context.WriteContext; import org.apache.fory.platform.AndroidSupport; +import org.apache.fory.platform.internal._JDKAccess; import org.apache.fory.resolver.TypeResolver; import org.apache.fory.util.Preconditions; import org.apache.fory.util.function.SerializableFunction; -import org.apache.fory.util.unsafe._JDKAccess; /** * Serializer for java serializable lambda. Use fory to serialize java lambda instead of JDK diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectStreamSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectStreamSerializer.java index fabaa64afd..a5f1fe852a 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectStreamSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectStreamSerializer.java @@ -62,6 +62,7 @@ import org.apache.fory.meta.TypeDef; import org.apache.fory.platform.AndroidSupport; import org.apache.fory.platform.GraalvmSupport; +import org.apache.fory.platform.internal._JDKAccess; import org.apache.fory.reflect.ObjectCreator; import org.apache.fory.reflect.ObjectCreators; import org.apache.fory.reflect.ReflectionUtils; @@ -74,7 +75,6 @@ import org.apache.fory.type.Types; import org.apache.fory.util.ExceptionUtils; import org.apache.fory.util.Preconditions; -import org.apache.fory.util.unsafe._JDKAccess; /** * Implement jdk custom serialization only if following conditions are met: diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/ReplaceResolveSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/ReplaceResolveSerializer.java index 1556ba73ef..03d6492efa 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/ReplaceResolveSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/ReplaceResolveSerializer.java @@ -37,13 +37,13 @@ import org.apache.fory.logging.LoggerFactory; import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.platform.AndroidSupport; +import org.apache.fory.platform.internal._JDKAccess; import org.apache.fory.reflect.ReflectionUtils; import org.apache.fory.resolver.ClassResolver; import org.apache.fory.resolver.TypeInfo; import org.apache.fory.resolver.TypeResolver; import org.apache.fory.util.ExceptionUtils; import org.apache.fory.util.Preconditions; -import org.apache.fory.util.unsafe._JDKAccess; /** * Serializer for class which has jdk `writeReplace`/`readResolve` method defined. This serializer diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/SerializedLambdaSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/SerializedLambdaSerializer.java index e359979a08..ec8c3a604e 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/SerializedLambdaSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/SerializedLambdaSerializer.java @@ -28,9 +28,9 @@ import org.apache.fory.exception.ForyException; import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.platform.AndroidSupport; +import org.apache.fory.platform.internal._JDKAccess; import org.apache.fory.resolver.TypeResolver; import org.apache.fory.util.Preconditions; -import org.apache.fory.util.unsafe._JDKAccess; /** * Serializer for {@link SerializedLambda}. It writes the JDK lambda payload through the public diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/Serializers.java b/java/fory-core/src/main/java/org/apache/fory/serializer/Serializers.java index 8da69a667b..0f534f79bd 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/Serializers.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/Serializers.java @@ -51,6 +51,7 @@ import org.apache.fory.meta.TypeDef; import org.apache.fory.platform.AndroidSupport; import org.apache.fory.platform.GraalvmSupport; +import org.apache.fory.platform.internal._JDKAccess; import org.apache.fory.reflect.ReflectionUtils; import org.apache.fory.resolver.ClassResolver; import org.apache.fory.resolver.TypeInfo; @@ -66,7 +67,6 @@ import org.apache.fory.serializer.scala.SingletonObjectSerializer; import org.apache.fory.util.ExceptionUtils; import org.apache.fory.util.StringUtils; -import org.apache.fory.util.unsafe._JDKAccess; /** Serialization utils and common serializers. */ @SuppressWarnings({"rawtypes", "unchecked"}) diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/StringSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/StringSerializer.java index 5d423a44fb..f2fcac8805 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/StringSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/StringSerializer.java @@ -45,11 +45,11 @@ import org.apache.fory.platform.AndroidSupport; import org.apache.fory.platform.JdkVersion; import org.apache.fory.platform.UnsafeOps; +import org.apache.fory.platform.internal._JDKAccess; import org.apache.fory.util.MathUtils; import org.apache.fory.util.Preconditions; import org.apache.fory.util.StringEncodingUtils; import org.apache.fory.util.StringUtils; -import org.apache.fory.util.unsafe._JDKAccess; /** * String serializer based on {@link sun.misc.Unsafe} and {@link MethodHandle} for speed. diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/CollectionSerializers.java b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/CollectionSerializers.java index 853c9071f9..16a420fd97 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/CollectionSerializers.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/CollectionSerializers.java @@ -60,6 +60,7 @@ import org.apache.fory.memory.MemoryUtils; import org.apache.fory.platform.GraalvmSupport; import org.apache.fory.platform.UnsafeOps; +import org.apache.fory.platform.internal._JDKAccess; import org.apache.fory.reflect.FieldAccessor; import org.apache.fory.reflect.ReflectionUtils; import org.apache.fory.resolver.ClassResolver; @@ -71,7 +72,6 @@ import org.apache.fory.serializer.Serializer; import org.apache.fory.serializer.Serializers; import org.apache.fory.util.Preconditions; -import org.apache.fory.util.unsafe._JDKAccess; /** * Serializers for classes implements {@link Collection}. All collection serializers should extend diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/ImmutableCollectionSerializers.java b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/ImmutableCollectionSerializers.java index 08c40435ed..a5b4dc539e 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/ImmutableCollectionSerializers.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/ImmutableCollectionSerializers.java @@ -34,9 +34,9 @@ import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.platform.AndroidSupport; import org.apache.fory.platform.JdkVersion; +import org.apache.fory.platform.internal._JDKAccess; import org.apache.fory.resolver.TypeResolver; import org.apache.fory.util.ExceptionUtils; -import org.apache.fory.util.unsafe._JDKAccess; /** Serializers for jdk9+ java.util.ImmutableCollections. */ @SuppressWarnings({"unchecked", "rawtypes"}) diff --git a/java/fory-core/src/main/java/org/apache/fory/util/ClassLoaderUtils.java b/java/fory-core/src/main/java/org/apache/fory/util/ClassLoaderUtils.java index ac5a382297..4f5af9859b 100644 --- a/java/fory-core/src/main/java/org/apache/fory/util/ClassLoaderUtils.java +++ b/java/fory-core/src/main/java/org/apache/fory/util/ClassLoaderUtils.java @@ -37,7 +37,7 @@ import org.apache.fory.logging.Logger; import org.apache.fory.logging.LoggerFactory; import org.apache.fory.platform.AndroidSupport; -import org.apache.fory.util.unsafe.DefineClass; +import org.apache.fory.platform.internal.DefineClass; /** ClassLoader utility for defining class and loading class by strategies. */ public class ClassLoaderUtils { diff --git a/java/fory-core/src/main/java/org/apache/fory/util/DefaultValueUtils.java b/java/fory-core/src/main/java/org/apache/fory/util/DefaultValueUtils.java index a6713e8717..59c3088862 100644 --- a/java/fory-core/src/main/java/org/apache/fory/util/DefaultValueUtils.java +++ b/java/fory-core/src/main/java/org/apache/fory/util/DefaultValueUtils.java @@ -34,12 +34,12 @@ import org.apache.fory.logging.Logger; import org.apache.fory.logging.LoggerFactory; import org.apache.fory.platform.AndroidSupport; +import org.apache.fory.platform.internal._JDKAccess; import org.apache.fory.reflect.FieldAccessor; import org.apache.fory.resolver.TypeResolver; import org.apache.fory.type.ScalaTypes; import org.apache.fory.type.TypeUtils; import org.apache.fory.type.Types; -import org.apache.fory.util.unsafe._JDKAccess; /** * Utility class for detecting Scala classes with default values and their default value methods. diff --git a/java/fory-core/src/main/java/org/apache/fory/util/function/Functions.java b/java/fory-core/src/main/java/org/apache/fory/util/function/Functions.java index 23c8d14270..b705b606ff 100644 --- a/java/fory-core/src/main/java/org/apache/fory/util/function/Functions.java +++ b/java/fory-core/src/main/java/org/apache/fory/util/function/Functions.java @@ -35,11 +35,11 @@ import org.apache.fory.exception.ForyException; import org.apache.fory.platform.AndroidSupport; import org.apache.fory.platform.GraalvmSupport; +import org.apache.fory.platform.internal._JDKAccess; import org.apache.fory.reflect.ReflectionUtils; import org.apache.fory.util.Preconditions; import org.apache.fory.util.record.RecordComponent; import org.apache.fory.util.record.RecordUtils; -import org.apache.fory.util.unsafe._JDKAccess; /** Utility for lambda functions. */ public class Functions { diff --git a/java/fory-core/src/main/java/org/apache/fory/util/record/RecordUtils.java b/java/fory-core/src/main/java/org/apache/fory/util/record/RecordUtils.java index e7dd11fd3a..215a92152e 100644 --- a/java/fory-core/src/main/java/org/apache/fory/util/record/RecordUtils.java +++ b/java/fory-core/src/main/java/org/apache/fory/util/record/RecordUtils.java @@ -35,7 +35,7 @@ import org.apache.fory.collection.Tuple2; import org.apache.fory.exception.ForyException; import org.apache.fory.platform.AndroidSupport; -import org.apache.fory.util.unsafe._JDKAccess; +import org.apache.fory.platform.internal._JDKAccess; /** Utils for java.lang.Record. */ @SuppressWarnings({"rawtypes"}) diff --git a/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties b/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties index 3df07eed99..9ffc90564f 100644 --- a/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties +++ b/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties @@ -534,9 +534,9 @@ Args=--initialize-at-build-time=org.apache.fory.annotation.ForyField$Dynamic,\ org.apache.fory.util.record.RecordUtils$2,\ org.apache.fory.util.record.RecordUtils$3,\ org.apache.fory.util.record.RecordUtils,\ - org.apache.fory.util.unsafe._JDKAccess$1,\ - org.apache.fory.util.unsafe._JDKAccess,\ - org.apache.fory.util.unsafe._Lookup,\ + org.apache.fory.platform.internal._JDKAccess$1,\ + org.apache.fory.platform.internal._JDKAccess,\ + org.apache.fory.platform.internal._Lookup,\ org.apache.fory.codegen.JaninoUtils$CodeStats,\ org.apache.fory.shaded.org.codehaus.commons.compiler.Location,\ org.apache.fory.shaded.org.codehaus.commons.compiler.util.iterator.Iterables,\ diff --git a/java/fory-core/src/test/java/org/apache/fory/TestUtils.java b/java/fory-core/src/test/java/org/apache/fory/TestUtils.java index 4ea85a98ad..2a2e5f1e19 100644 --- a/java/fory-core/src/test/java/org/apache/fory/TestUtils.java +++ b/java/fory-core/src/test/java/org/apache/fory/TestUtils.java @@ -32,10 +32,10 @@ import org.apache.fory.collection.Tuple3; import org.apache.fory.meta.TypeDef; import org.apache.fory.platform.UnsafeOps; +import org.apache.fory.platform.internal._JDKAccess; import org.apache.fory.reflect.FieldAccessor; import org.apache.fory.reflect.ReflectionUtils; import org.apache.fory.type.Descriptor; -import org.apache.fory.util.unsafe._JDKAccess; import org.testng.SkipException; /** Test utils. */ diff --git a/java/fory-core/src/test/java/org/apache/fory/collection/MultiKeyWeakMapTest.java b/java/fory-core/src/test/java/org/apache/fory/collection/MultiKeyWeakMapTest.java index 5df82373cf..8dc2cba0da 100644 --- a/java/fory-core/src/test/java/org/apache/fory/collection/MultiKeyWeakMapTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/collection/MultiKeyWeakMapTest.java @@ -24,7 +24,7 @@ import java.util.Set; import org.apache.fory.logging.Logger; import org.apache.fory.logging.LoggerFactory; -import org.apache.fory.util.unsafe._JDKAccess; +import org.apache.fory.platform.internal._JDKAccess; import org.testng.SkipException; import org.testng.annotations.Test; diff --git a/java/fory-core/src/test/java/org/apache/fory/util/unsafe/DefineClassTest.java b/java/fory-core/src/test/java/org/apache/fory/platform/internal/DefineClassTest.java similarity index 98% rename from java/fory-core/src/test/java/org/apache/fory/util/unsafe/DefineClassTest.java rename to java/fory-core/src/test/java/org/apache/fory/platform/internal/DefineClassTest.java index c974bdd2b9..3e05a00792 100644 --- a/java/fory-core/src/test/java/org/apache/fory/util/unsafe/DefineClassTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/platform/internal/DefineClassTest.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.fory.util.unsafe; +package org.apache.fory.platform.internal; import java.util.Collections; import org.apache.fory.codegen.CompileUnit; diff --git a/java/fory-core/src/test/java/org/apache/fory/util/unsafe/JDKAccessTest.java b/java/fory-core/src/test/java/org/apache/fory/platform/internal/JDKAccessTest.java similarity index 99% rename from java/fory-core/src/test/java/org/apache/fory/util/unsafe/JDKAccessTest.java rename to java/fory-core/src/test/java/org/apache/fory/platform/internal/JDKAccessTest.java index a6565c0972..5b4a902b68 100644 --- a/java/fory-core/src/test/java/org/apache/fory/util/unsafe/JDKAccessTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/platform/internal/JDKAccessTest.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.fory.util.unsafe; +package org.apache.fory.platform.internal; import java.lang.invoke.MethodHandle; import java.lang.invoke.MethodHandles; diff --git a/java/fory-core/src/test/java/org/apache/fory/serializer/StringSerializerTest.java b/java/fory-core/src/test/java/org/apache/fory/serializer/StringSerializerTest.java index b621677bad..3ce6a7f2be 100644 --- a/java/fory-core/src/test/java/org/apache/fory/serializer/StringSerializerTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/serializer/StringSerializerTest.java @@ -36,9 +36,9 @@ import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.memory.MemoryUtils; import org.apache.fory.platform.JdkVersion; +import org.apache.fory.platform.internal._JDKAccess; import org.apache.fory.util.MathUtils; import org.apache.fory.util.StringUtils; -import org.apache.fory.util.unsafe._JDKAccess; import org.testng.Assert; import org.testng.SkipException; import org.testng.annotations.DataProvider; diff --git a/java/fory-extensions/src/main/java/org/apache/fory/extension/serializer/ProtobufSerializer.java b/java/fory-extensions/src/main/java/org/apache/fory/extension/serializer/ProtobufSerializer.java index 194967d687..9711e869a1 100644 --- a/java/fory-extensions/src/main/java/org/apache/fory/extension/serializer/ProtobufSerializer.java +++ b/java/fory-extensions/src/main/java/org/apache/fory/extension/serializer/ProtobufSerializer.java @@ -32,10 +32,10 @@ import org.apache.fory.context.ReadContext; import org.apache.fory.context.WriteContext; import org.apache.fory.memory.MemoryBuffer; +import org.apache.fory.platform.internal._JDKAccess; import org.apache.fory.serializer.Serializer; import org.apache.fory.serializer.Shareable; import org.apache.fory.util.ExceptionUtils; -import org.apache.fory.util.unsafe._JDKAccess; @SuppressWarnings({"rawtypes", "unchecked"}) public class ProtobufSerializer extends Serializer implements Shareable { diff --git a/java/fory-format/src/main/java/org/apache/fory/format/type/CustomTypeEncoderRegistry.java b/java/fory-format/src/main/java/org/apache/fory/format/type/CustomTypeEncoderRegistry.java index b952087970..f1c97f7a82 100644 --- a/java/fory-format/src/main/java/org/apache/fory/format/type/CustomTypeEncoderRegistry.java +++ b/java/fory-format/src/main/java/org/apache/fory/format/type/CustomTypeEncoderRegistry.java @@ -32,8 +32,8 @@ import org.apache.fory.codegen.JaninoUtils; import org.apache.fory.format.encoder.CustomCodec; import org.apache.fory.format.encoder.CustomCollectionFactory; +import org.apache.fory.platform.internal.DefineClass; import org.apache.fory.reflect.TypeRef; -import org.apache.fory.util.unsafe.DefineClass; /** * Keep a registry of custom codecs and collection factories. In order to deliver peak performance, diff --git a/scala/src/main/scala/org/apache/fory/serializer/scala/RangeSerializer.scala b/scala/src/main/scala/org/apache/fory/serializer/scala/RangeSerializer.scala index 70d5fed928..3fa114f104 100644 --- a/scala/src/main/scala/org/apache/fory/serializer/scala/RangeSerializer.scala +++ b/scala/src/main/scala/org/apache/fory/serializer/scala/RangeSerializer.scala @@ -27,7 +27,7 @@ import org.apache.fory.serializer.Serializer import org.apache.fory.serializer.collection.CollectionLikeSerializer import org.apache.fory.resolver.TypeResolver import java.util -import org.apache.fory.util.unsafe._JDKAccess +import org.apache.fory.platform.internal._JDKAccess import java.lang.invoke.{MethodHandle, MethodHandles} import scala.collection.immutable.NumericRange From 26dec9d6a03d72c717bb3bc4e8c15e4a8d81bb14 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sat, 23 May 2026 23:51:51 +0800 Subject: [PATCH 12/69] fix(java): add JDK25 unsafe replacements --- .../org/apache/fory/platform/UnsafeOps.java | 419 ++++++++++++++++ .../fory/platform/internal/_JDKAccess.java | 470 ++++++++++++++++++ .../fory/platform/internal/_Lookup.java | 69 +++ 3 files changed, 958 insertions(+) create mode 100644 java/fory-core/src/main/java25/org/apache/fory/platform/UnsafeOps.java create mode 100644 java/fory-core/src/main/java25/org/apache/fory/platform/internal/_JDKAccess.java create mode 100644 java/fory-core/src/main/java25/org/apache/fory/platform/internal/_Lookup.java diff --git a/java/fory-core/src/main/java25/org/apache/fory/platform/UnsafeOps.java b/java/fory-core/src/main/java25/org/apache/fory/platform/UnsafeOps.java new file mode 100644 index 0000000000..ce00b9b6c7 --- /dev/null +++ b/java/fory-core/src/main/java25/org/apache/fory/platform/UnsafeOps.java @@ -0,0 +1,419 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.platform; + +import java.lang.invoke.MethodHandles; +import java.lang.invoke.VarHandle; +import java.lang.reflect.Field; +import java.nio.ByteOrder; +import org.apache.fory.annotation.Internal; +import org.apache.fory.platform.internal._JDKAccess; +import sun.misc.Unsafe; + +/** A utility class for array memory operations on JDK25+. */ +@Internal +@SuppressWarnings("restriction") +public final class UnsafeOps { + @SuppressWarnings("restriction") + public static final Unsafe UNSAFE = _JDKAccess.UNSAFE; + + public static final int BOOLEAN_ARRAY_OFFSET = 0; + public static final int BYTE_ARRAY_OFFSET = 0; + public static final int CHAR_ARRAY_OFFSET = 0; + public static final int SHORT_ARRAY_OFFSET = 0; + public static final int INT_ARRAY_OFFSET = 0; + public static final int LONG_ARRAY_OFFSET = 0; + public static final int FLOAT_ARRAY_OFFSET = 0; + public static final int DOUBLE_ARRAY_OFFSET = 0; + private static final boolean BIG_ENDIAN = ByteOrder.nativeOrder() == ByteOrder.BIG_ENDIAN; + private static final VarHandle BYTE_ARRAY_CHAR = + MethodHandles.byteArrayViewVarHandle(char[].class, ByteOrder.nativeOrder()); + private static final VarHandle BYTE_ARRAY_SHORT = + MethodHandles.byteArrayViewVarHandle(short[].class, ByteOrder.nativeOrder()); + private static final VarHandle BYTE_ARRAY_INT = + MethodHandles.byteArrayViewVarHandle(int[].class, ByteOrder.nativeOrder()); + private static final VarHandle BYTE_ARRAY_LONG = + MethodHandles.byteArrayViewVarHandle(long[].class, ByteOrder.nativeOrder()); + private static final VarHandle BYTE_ARRAY_FLOAT = + MethodHandles.byteArrayViewVarHandle(float[].class, ByteOrder.nativeOrder()); + private static final VarHandle BYTE_ARRAY_DOUBLE = + MethodHandles.byteArrayViewVarHandle(double[].class, ByteOrder.nativeOrder()); + private static final boolean unaligned; + + private UnsafeOps() {} + + static { + String arch = System.getProperty("os.arch", ""); + if ("ppc64le".equals(arch) || "ppc64".equals(arch) || "s390x".equals(arch)) { + unaligned = true; + } else { + unaligned = arch.matches("^(i[3-6]86|x86(_64)?|x64|amd64|aarch64)$"); + } + } + + /** + * Returns true when the underlying system is known to support unaligned access. JDK25 array + * accessors do not use Unsafe, but callers keep this gate for vectorized array scans. + */ + public static boolean unaligned() { + return unaligned; + } + + public static long objectFieldOffset(Field f) { + throw unsupportedObjectMemory(); + } + + public static int getInt(Object object, long offset) { + if (object instanceof byte[]) { + return (int) BYTE_ARRAY_INT.get((byte[]) object, toIntIndex(offset)); + } + return getIntFromArray(object, offset); + } + + public static void putInt(Object object, long offset, int value) { + if (object instanceof byte[]) { + BYTE_ARRAY_INT.set((byte[]) object, toIntIndex(offset), value); + return; + } + putIntToArray(object, offset, value); + } + + public static boolean getBoolean(Object object, long offset) { + if (object instanceof boolean[]) { + return ((boolean[]) object)[toIntIndex(offset)]; + } + return getByte(object, offset) != 0; + } + + public static void putBoolean(Object object, long offset, boolean value) { + if (object instanceof boolean[]) { + ((boolean[]) object)[toIntIndex(offset)] = value; + return; + } + putByte(object, offset, value ? (byte) 1 : (byte) 0); + } + + public static byte getByte(Object object, long offset) { + return getArrayByte(object, offset); + } + + public static void putByte(Object object, long offset, byte value) { + putArrayByte(object, offset, value); + } + + public static short getShort(Object object, long offset) { + if (object instanceof byte[]) { + return (short) BYTE_ARRAY_SHORT.get((byte[]) object, toIntIndex(offset)); + } + return (short) getIntN(object, offset, Short.BYTES); + } + + public static void putShort(Object object, long offset, short value) { + if (object instanceof byte[]) { + BYTE_ARRAY_SHORT.set((byte[]) object, toIntIndex(offset), value); + return; + } + putIntN(object, offset, value, Short.BYTES); + } + + public static char getChar(Object obj, long offset) { + if (obj instanceof byte[]) { + return (char) BYTE_ARRAY_CHAR.get((byte[]) obj, toIntIndex(offset)); + } + return (char) getIntN(obj, offset, Character.BYTES); + } + + public static void putChar(Object obj, long offset, char value) { + if (obj instanceof byte[]) { + BYTE_ARRAY_CHAR.set((byte[]) obj, toIntIndex(offset), value); + return; + } + putIntN(obj, offset, value, Character.BYTES); + } + + public static long getLong(Object object, long offset) { + if (object instanceof byte[]) { + return (long) BYTE_ARRAY_LONG.get((byte[]) object, toIntIndex(offset)); + } + long value = 0; + if (BIG_ENDIAN) { + for (int i = 0; i < Long.BYTES; i++) { + value = (value << Byte.SIZE) | (getArrayByte(object, offset + i) & 0xffL); + } + } else { + for (int i = Long.BYTES - 1; i >= 0; i--) { + value = (value << Byte.SIZE) | (getArrayByte(object, offset + i) & 0xffL); + } + } + return value; + } + + public static void putLong(Object object, long offset, long value) { + if (object instanceof byte[]) { + BYTE_ARRAY_LONG.set((byte[]) object, toIntIndex(offset), value); + return; + } + if (BIG_ENDIAN) { + for (int i = Long.BYTES - 1; i >= 0; i--) { + putArrayByte(object, offset + i, (byte) value); + value >>>= Byte.SIZE; + } + } else { + for (int i = 0; i < Long.BYTES; i++) { + putArrayByte(object, offset + i, (byte) value); + value >>>= Byte.SIZE; + } + } + } + + public static float getFloat(Object object, long offset) { + if (object instanceof byte[]) { + return (float) BYTE_ARRAY_FLOAT.get((byte[]) object, toIntIndex(offset)); + } + return Float.intBitsToFloat(getInt(object, offset)); + } + + public static void putFloat(Object object, long offset, float value) { + if (object instanceof byte[]) { + BYTE_ARRAY_FLOAT.set((byte[]) object, toIntIndex(offset), value); + return; + } + putInt(object, offset, Float.floatToRawIntBits(value)); + } + + public static double getDouble(Object object, long offset) { + if (object instanceof byte[]) { + return (double) BYTE_ARRAY_DOUBLE.get((byte[]) object, toIntIndex(offset)); + } + return Double.longBitsToDouble(getLong(object, offset)); + } + + public static void putDouble(Object object, long offset, double value) { + if (object instanceof byte[]) { + BYTE_ARRAY_DOUBLE.set((byte[]) object, toIntIndex(offset), value); + return; + } + putLong(object, offset, Double.doubleToRawLongBits(value)); + } + + public static Object getObject(Object o, long offset) { + throw unsupportedObjectMemory(); + } + + public static void putObject(Object object, long offset, Object value) { + throw unsupportedObjectMemory(); + } + + public static void copyMemory( + Object src, long srcOffset, Object dst, long dstOffset, long length) { + if (src == null || dst == null) { + throw unsupportedNativeMemory(); + } + if (length < 0) { + throw new IllegalArgumentException("length must be non-negative: " + length); + } + int len = toIntLength(length); + if (src instanceof byte[] && dst instanceof byte[]) { + System.arraycopy((byte[]) src, toIntIndex(srcOffset), (byte[]) dst, toIntIndex(dstOffset), len); + return; + } + if (!isPrimitiveArray(src) || !isPrimitiveArray(dst)) { + throw unsupportedObjectMemory(); + } + if (src == dst && srcOffset < dstOffset && dstOffset < srcOffset + length) { + for (long i = length - 1; i >= 0; i--) { + putArrayByte(dst, dstOffset + i, getArrayByte(src, srcOffset + i)); + } + } else { + for (long i = 0; i < length; i++) { + putArrayByte(dst, dstOffset + i, getArrayByte(src, srcOffset + i)); + } + } + } + + /** Create an instance of type. This method does not call constructor. */ + public static T newInstance(Class type) { + throw new UnsupportedOperationException( + "Constructor-bypassing allocation is unsupported on JDK25 without sun.misc.Unsafe"); + } + + private static int getIntFromArray(Object object, long offset) { + return getIntN(object, offset, Integer.BYTES); + } + + private static void putIntToArray(Object object, long offset, int value) { + putIntN(object, offset, value, Integer.BYTES); + } + + private static int getIntN(Object object, long offset, int bytes) { + int value = 0; + if (BIG_ENDIAN) { + for (int i = 0; i < bytes; i++) { + value = (value << Byte.SIZE) | (getArrayByte(object, offset + i) & 0xff); + } + } else { + for (int i = bytes - 1; i >= 0; i--) { + value = (value << Byte.SIZE) | (getArrayByte(object, offset + i) & 0xff); + } + } + return value; + } + + private static void putIntN(Object object, long offset, int value, int bytes) { + if (BIG_ENDIAN) { + for (int i = bytes - 1; i >= 0; i--) { + putArrayByte(object, offset + i, (byte) value); + value >>>= Byte.SIZE; + } + } else { + for (int i = 0; i < bytes; i++) { + putArrayByte(object, offset + i, (byte) value); + value >>>= Byte.SIZE; + } + } + } + + private static byte getArrayByte(Object object, long offset) { + checkOffset(offset); + if (object instanceof byte[]) { + return ((byte[]) object)[toIntIndex(offset)]; + } else if (object instanceof boolean[]) { + return ((boolean[]) object)[toIntIndex(offset)] ? (byte) 1 : (byte) 0; + } else if (object instanceof char[]) { + return getIntByte( + ((char[]) object)[toIntIndex(offset / Character.BYTES)], offset, Character.BYTES); + } else if (object instanceof short[]) { + return getIntByte(((short[]) object)[toIntIndex(offset / Short.BYTES)], offset, Short.BYTES); + } else if (object instanceof int[]) { + return getIntByte( + ((int[]) object)[toIntIndex(offset / Integer.BYTES)], offset, Integer.BYTES); + } else if (object instanceof long[]) { + return getLongByte(((long[]) object)[toIntIndex(offset / Long.BYTES)], offset); + } else if (object instanceof float[]) { + int value = Float.floatToRawIntBits(((float[]) object)[toIntIndex(offset / Float.BYTES)]); + return getIntByte(value, offset, Float.BYTES); + } else if (object instanceof double[]) { + long value = + Double.doubleToRawLongBits(((double[]) object)[toIntIndex(offset / Double.BYTES)]); + return getLongByte(value, offset); + } + throw unsupportedObjectMemory(); + } + + private static void putArrayByte(Object object, long offset, byte value) { + checkOffset(offset); + if (object instanceof byte[]) { + ((byte[]) object)[toIntIndex(offset)] = value; + } else if (object instanceof boolean[]) { + ((boolean[]) object)[toIntIndex(offset)] = value != 0; + } else if (object instanceof char[]) { + char[] array = (char[]) object; + int index = toIntIndex(offset / Character.BYTES); + array[index] = (char) setIntByte(array[index], offset, value, Character.BYTES); + } else if (object instanceof short[]) { + short[] array = (short[]) object; + int index = toIntIndex(offset / Short.BYTES); + array[index] = (short) setIntByte(array[index], offset, value, Short.BYTES); + } else if (object instanceof int[]) { + int[] array = (int[]) object; + int index = toIntIndex(offset / Integer.BYTES); + array[index] = setIntByte(array[index], offset, value, Integer.BYTES); + } else if (object instanceof long[]) { + long[] array = (long[]) object; + int index = toIntIndex(offset / Long.BYTES); + array[index] = setLongByte(array[index], offset, value); + } else if (object instanceof float[]) { + float[] array = (float[]) object; + int index = toIntIndex(offset / Float.BYTES); + int bits = Float.floatToRawIntBits(array[index]); + array[index] = Float.intBitsToFloat(setIntByte(bits, offset, value, Float.BYTES)); + } else if (object instanceof double[]) { + double[] array = (double[]) object; + int index = toIntIndex(offset / Double.BYTES); + long bits = Double.doubleToRawLongBits(array[index]); + array[index] = Double.longBitsToDouble(setLongByte(bits, offset, value)); + } else { + throw unsupportedObjectMemory(); + } + } + + private static byte getIntByte(int value, long offset, int width) { + int shift = byteShift(offset, width); + return (byte) (value >>> shift); + } + + private static byte getLongByte(long value, long offset) { + int shift = byteShift(offset, Long.BYTES); + return (byte) (value >>> shift); + } + + private static int setIntByte(int oldValue, long offset, byte value, int width) { + int shift = byteShift(offset, width); + int mask = 0xff << shift; + return (oldValue & ~mask) | ((value & 0xff) << shift); + } + + private static long setLongByte(long oldValue, long offset, byte value) { + int shift = byteShift(offset, Long.BYTES); + long mask = 0xffL << shift; + return (oldValue & ~mask) | ((long) (value & 0xff) << shift); + } + + private static int byteShift(long offset, int width) { + int byteIndex = (int) Math.floorMod(offset, width); + return (BIG_ENDIAN ? width - 1 - byteIndex : byteIndex) * Byte.SIZE; + } + + private static boolean isPrimitiveArray(Object object) { + Class cls = object.getClass(); + return cls.isArray() && cls.getComponentType().isPrimitive(); + } + + private static int toIntIndex(long offset) { + if (offset < 0 || offset > Integer.MAX_VALUE) { + throw new IndexOutOfBoundsException("offset out of int range: " + offset); + } + return (int) offset; + } + + private static int toIntLength(long length) { + if (length > Integer.MAX_VALUE) { + throw new IndexOutOfBoundsException("length out of int range: " + length); + } + return (int) length; + } + + private static void checkOffset(long offset) { + if (offset < 0) { + throw new IndexOutOfBoundsException("offset must be non-negative: " + offset); + } + } + + private static UnsupportedOperationException unsupportedObjectMemory() { + return new UnsupportedOperationException( + "Object field and reference-offset memory access is unsupported on JDK25 without " + + "sun.misc.Unsafe"); + } + + private static UnsupportedOperationException unsupportedNativeMemory() { + return new UnsupportedOperationException( + "Raw native-address memory access is unsupported on JDK25 without sun.misc.Unsafe"); + } +} diff --git a/java/fory-core/src/main/java25/org/apache/fory/platform/internal/_JDKAccess.java b/java/fory-core/src/main/java25/org/apache/fory/platform/internal/_JDKAccess.java new file mode 100644 index 0000000000..7bdb2ac158 --- /dev/null +++ b/java/fory-core/src/main/java25/org/apache/fory/platform/internal/_JDKAccess.java @@ -0,0 +1,470 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.platform.internal; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.lang.invoke.CallSite; +import java.lang.invoke.LambdaMetafactory; +import java.lang.invoke.MethodHandle; +import java.lang.invoke.MethodHandles; +import java.lang.invoke.MethodHandles.Lookup; +import java.lang.invoke.MethodType; +import java.lang.invoke.VarHandle; +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.util.HashMap; +import java.util.Map; +import java.util.function.BiConsumer; +import java.util.function.Consumer; +import java.util.function.Function; +import java.util.function.Predicate; +import java.util.function.ToDoubleFunction; +import java.util.function.ToIntFunction; +import java.util.function.ToLongFunction; +import org.apache.fory.collection.ClassValueCache; +import org.apache.fory.collection.Tuple2; +import org.apache.fory.memory.MemoryBuffer; +import org.apache.fory.platform.GraalvmSupport; +import org.apache.fory.platform.JdkVersion; +import org.apache.fory.type.TypeUtils; +import org.apache.fory.util.ExceptionUtils; +import org.apache.fory.util.Preconditions; +import org.apache.fory.util.function.ToByteFunction; +import org.apache.fory.util.function.ToCharFunction; +import org.apache.fory.util.function.ToFloatFunction; +import org.apache.fory.util.function.ToShortFunction; +import sun.misc.Unsafe; + +/** JDK internals access for the JDK25 multi-release runtime. */ +// CHECKSTYLE.OFF:TypeName +public class _JDKAccess { + // CHECKSTYLE.ON:TypeName + public static final boolean IS_OPEN_J9; + public static final Unsafe UNSAFE = null; + public static final boolean JDK_INTERNAL_FIELD_ACCESS; + public static final Class _INNER_UNSAFE_CLASS = null; + public static final Object _INNER_UNSAFE = null; + + private static final ClassValueCache lookupCache = ClassValueCache.newClassKeyCache(32); + + public static final boolean STRING_VALUE_FIELD_IS_CHARS; + public static final boolean STRING_VALUE_FIELD_IS_BYTES; + public static final boolean STRING_HAS_COUNT_OFFSET; + private static final long STRING_VALUE_FIELD_OFFSET = -1; + private static final long STRING_COUNT_FIELD_OFFSET = -1; + private static final long STRING_OFFSET_FIELD_OFFSET = -1; + private static final VarHandle STRING_VALUE_HANDLE; + private static final VarHandle STRING_CODER_HANDLE; + private static final VarHandle STRING_COUNT_HANDLE; + private static final VarHandle STRING_OFFSET_HANDLE; + private static final VarHandle BAS_BUF_HANDLE; + private static final VarHandle BAS_COUNT_HANDLE; + private static final VarHandle BIS_BUF_HANDLE; + private static final VarHandle BIS_POS_HANDLE; + private static final VarHandle BIS_COUNT_HANDLE; + + static { + String jmvName = System.getProperty("java.vm.name", ""); + IS_OPEN_J9 = jmvName.contains("OpenJ9"); + } + + static { + try { + Field valueField = String.class.getDeclaredField("value"); + STRING_VALUE_FIELD_IS_CHARS = valueField.getType() == char[].class; + STRING_VALUE_FIELD_IS_BYTES = valueField.getType() == byte[].class; + Field countField = getStringFieldNullable("count"); + Field offsetField = getStringFieldNullable("offset"); + if (countField != null || offsetField != null) { + Preconditions.checkArgument( + countField != null && offsetField != null, "Current jdk not supported"); + Preconditions.checkArgument( + countField.getType() == int.class && offsetField.getType() == int.class, + "Current jdk not supported"); + STRING_HAS_COUNT_OFFSET = true; + } else { + STRING_HAS_COUNT_OFFSET = false; + } + FieldHandles handles = initFieldHandles(valueField.getType(), countField, offsetField); + JDK_INTERNAL_FIELD_ACCESS = handles != null; + STRING_VALUE_HANDLE = handles == null ? null : handles.stringValue; + STRING_CODER_HANDLE = handles == null ? null : handles.stringCoder; + STRING_COUNT_HANDLE = handles == null ? null : handles.stringCount; + STRING_OFFSET_HANDLE = handles == null ? null : handles.stringOffset; + BAS_BUF_HANDLE = handles == null ? null : handles.basBuf; + BAS_COUNT_HANDLE = handles == null ? null : handles.basCount; + BIS_BUF_HANDLE = handles == null ? null : handles.bisBuf; + BIS_POS_HANDLE = handles == null ? null : handles.bisPos; + BIS_COUNT_HANDLE = handles == null ? null : handles.bisCount; + } catch (NoSuchFieldException e) { + throw new RuntimeException(e); + } + } + + private static Field getStringFieldNullable(String fieldName) { + try { + return String.class.getDeclaredField(fieldName); + } catch (NoSuchFieldException e) { + return null; + } + } + + private static FieldHandles initFieldHandles( + Class stringValueType, Field countField, Field offsetField) { + try { + Lookup stringLookup = MethodHandles.privateLookupIn(String.class, MethodHandles.lookup()); + VarHandle stringValue = stringLookup.findVarHandle(String.class, "value", stringValueType); + VarHandle stringCoder = + STRING_VALUE_FIELD_IS_BYTES + ? stringLookup.findVarHandle(String.class, "coder", byte.class) + : null; + VarHandle stringCount = + countField == null ? null : stringLookup.findVarHandle(String.class, "count", int.class); + VarHandle stringOffset = + offsetField == null + ? null + : stringLookup.findVarHandle(String.class, "offset", int.class); + Lookup basLookup = + MethodHandles.privateLookupIn(ByteArrayOutputStream.class, MethodHandles.lookup()); + Lookup bisLookup = + MethodHandles.privateLookupIn(ByteArrayInputStream.class, MethodHandles.lookup()); + return new FieldHandles( + stringValue, + stringCoder, + stringCount, + stringOffset, + basLookup.findVarHandle(ByteArrayOutputStream.class, "buf", byte[].class), + basLookup.findVarHandle(ByteArrayOutputStream.class, "count", int.class), + bisLookup.findVarHandle(ByteArrayInputStream.class, "buf", byte[].class), + bisLookup.findVarHandle(ByteArrayInputStream.class, "pos", int.class), + bisLookup.findVarHandle(ByteArrayInputStream.class, "count", int.class)); + } catch (Throwable ignored) { + return null; + } + } + + private static class FieldHandles { + private final VarHandle stringValue; + private final VarHandle stringCoder; + private final VarHandle stringCount; + private final VarHandle stringOffset; + private final VarHandle basBuf; + private final VarHandle basCount; + private final VarHandle bisBuf; + private final VarHandle bisPos; + private final VarHandle bisCount; + + private FieldHandles( + VarHandle stringValue, + VarHandle stringCoder, + VarHandle stringCount, + VarHandle stringOffset, + VarHandle basBuf, + VarHandle basCount, + VarHandle bisBuf, + VarHandle bisPos, + VarHandle bisCount) { + this.stringValue = stringValue; + this.stringCoder = stringCoder; + this.stringCount = stringCount; + this.stringOffset = stringOffset; + this.basBuf = basBuf; + this.basCount = basCount; + this.bisBuf = bisBuf; + this.bisPos = bisPos; + this.bisCount = bisCount; + } + } + + public static Object getStringValue(String value) { + checkInternalFieldAccess("String.value"); + return STRING_VALUE_HANDLE.get(value); + } + + public static byte getStringCoder(String value) { + checkInternalFieldAccess("String.coder"); + if (STRING_CODER_HANDLE == null) { + throw new UnsupportedOperationException("String.coder is not available on this JDK"); + } + return (byte) STRING_CODER_HANDLE.get(value); + } + + public static int getStringOffset(String value) { + checkInternalFieldAccess("String.offset"); + if (STRING_OFFSET_HANDLE == null) { + throw new UnsupportedOperationException("String.offset is not available on this JDK"); + } + return (int) STRING_OFFSET_HANDLE.get(value); + } + + public static int getStringCount(String value) { + checkInternalFieldAccess("String.count"); + if (STRING_COUNT_HANDLE == null) { + throw new UnsupportedOperationException("String.count is not available on this JDK"); + } + return (int) STRING_COUNT_HANDLE.get(value); + } + + // CHECKSTYLE.OFF:MethodName + + public static Lookup _trustedLookup(Class objectClass) { + // CHECKSTYLE.ON:MethodName + if (GraalvmSupport.isGraalBuildTime()) { + return _Lookup._trustedLookup(objectClass); + } + return lookupCache.get(objectClass, () -> _Lookup._trustedLookup(objectClass)); + } + + public static void wrap(ByteArrayOutputStream stream, MemoryBuffer buffer) { + Preconditions.checkNotNull(stream); + checkInternalFieldAccess("ByteArrayOutputStream"); + byte[] buf = (byte[]) BAS_BUF_HANDLE.get(stream); + int count = (int) BAS_COUNT_HANDLE.get(stream); + buffer.pointTo(buf, 0, buf.length); + buffer.writerIndex(count); + } + + public static void wrap(MemoryBuffer buffer, ByteArrayOutputStream stream) { + Preconditions.checkNotNull(stream); + checkInternalFieldAccess("ByteArrayOutputStream"); + byte[] bytes = buffer.getHeapMemory(); + Preconditions.checkNotNull(bytes); + BAS_BUF_HANDLE.set(stream, bytes); + BAS_COUNT_HANDLE.set(stream, buffer.writerIndex()); + } + + public static void wrap(ByteArrayInputStream stream, MemoryBuffer buffer) { + Preconditions.checkNotNull(stream); + checkInternalFieldAccess("ByteArrayInputStream"); + byte[] buf = (byte[]) BIS_BUF_HANDLE.get(stream); + int count = (int) BIS_COUNT_HANDLE.get(stream); + int pos = (int) BIS_POS_HANDLE.get(stream); + buffer.pointTo(buf, 0, count); + buffer.readerIndex(pos); + } + + private static void checkInternalFieldAccess(String target) { + if (!JDK_INTERNAL_FIELD_ACCESS) { + throw new UnsupportedOperationException( + target + + " private access is unavailable; open java.base/java.lang and java.base/java.io " + + "to org.apache.fory.core"); + } + } + + public static T tryMakeFunction( + Lookup lookup, MethodHandle handle, Class functionInterface) { + try { + return makeFunction(lookup, handle, functionInterface); + } catch (Throwable e) { + ExceptionUtils.ignore(e); + throw new IllegalStateException(); + } + } + + private static final MethodType jdkFunctionMethodType = + MethodType.methodType(Object.class, Object.class); + + @SuppressWarnings("unchecked") + public static Function makeJDKFunction(Lookup lookup, MethodHandle handle) { + return makeJDKFunction(lookup, handle, jdkFunctionMethodType); + } + + @SuppressWarnings("unchecked") + public static Function makeJDKFunction( + Lookup lookup, MethodHandle handle, MethodType methodType) { + try { + CallSite callSite = + LambdaMetafactory.metafactory( + lookup, + "apply", + MethodType.methodType(Function.class), + methodType, + handle, + boxedMethodType(handle.type())); + return (Function) callSite.getTarget().invokeExact(); + } catch (Throwable e) { + throw ExceptionUtils.throwException(e); + } + } + + private static final MethodType jdkConsumerMethodType = + MethodType.methodType(void.class, Object.class); + + @SuppressWarnings("unchecked") + public static Consumer makeJDKConsumer(Lookup lookup, MethodHandle handle) { + try { + CallSite callSite = + LambdaMetafactory.metafactory( + lookup, + "accept", + MethodType.methodType(Consumer.class), + jdkConsumerMethodType, + handle, + boxedMethodType(handle.type())); + return (Consumer) callSite.getTarget().invokeExact(); + } catch (Throwable e) { + throw ExceptionUtils.throwException(e); + } + } + + private static final MethodType jdkBiConsumerMethodType = + MethodType.methodType(void.class, Object.class, Object.class); + + @SuppressWarnings("unchecked") + public static BiConsumer makeJDKBiConsumer(Lookup lookup, MethodHandle handle) { + try { + CallSite callSite = + LambdaMetafactory.metafactory( + lookup, + "accept", + MethodType.methodType(BiConsumer.class), + jdkBiConsumerMethodType, + handle, + boxedMethodType(handle.type())); + return (BiConsumer) callSite.getTarget().invokeExact(); + } catch (Throwable e) { + throw ExceptionUtils.throwException(e); + } + } + + private static MethodType boxedMethodType(MethodType methodType) { + Class[] paramTypes = new Class[methodType.parameterCount()]; + for (int i = 0; i < paramTypes.length; i++) { + Class t = methodType.parameterType(i); + if (t.isPrimitive()) { + t = TypeUtils.wrap(t); + } + paramTypes[i] = t; + } + return MethodType.methodType(methodType.returnType(), paramTypes); + } + + @SuppressWarnings("unchecked") + public static T makeFunction(Lookup lookup, MethodHandle handle, Method methodToImpl) { + MethodType instantiatedMethodType = boxedMethodType(handle.type()); + MethodType methodToImplType = + MethodType.methodType(methodToImpl.getReturnType(), methodToImpl.getParameterTypes()); + try { + CallSite callSite = + LambdaMetafactory.metafactory( + lookup, + methodToImpl.getName(), + MethodType.methodType(methodToImpl.getDeclaringClass()), + methodToImplType, + handle, + instantiatedMethodType); + return (T) callSite.getTarget().invokeExact(); + } catch (Throwable e) { + throw ExceptionUtils.throwException(e); + } + } + + public static T makeFunction(Lookup lookup, MethodHandle handle, Class functionInterface) { + String invokedName = "apply"; + try { + Method method = null; + Method[] methods = functionInterface.getMethods(); + for (Method interfaceMethod : methods) { + if (interfaceMethod.getName().equals(invokedName)) { + method = interfaceMethod; + break; + } + } + if (method == null) { + Preconditions.checkArgument(methods.length == 1); + method = methods[0]; + invokedName = method.getName(); + } + MethodType interfaceType = + MethodType.methodType(method.getReturnType(), method.getParameterTypes()); + CallSite callSite = + LambdaMetafactory.metafactory( + lookup, + invokedName, + MethodType.methodType(functionInterface), + interfaceType, + handle, + interfaceType); + return (T) callSite.getTarget().invoke(); + } catch (Throwable e) { + throw ExceptionUtils.throwException(e); + } + } + + private static final Map, Tuple2, String>> methodMap = new HashMap<>(); + + static { + methodMap.put(boolean.class, Tuple2.of(Predicate.class, "test")); + methodMap.put(byte.class, Tuple2.of(ToByteFunction.class, "applyAsByte")); + methodMap.put(char.class, Tuple2.of(ToCharFunction.class, "applyAsChar")); + methodMap.put(short.class, Tuple2.of(ToShortFunction.class, "applyAsShort")); + methodMap.put(int.class, Tuple2.of(ToIntFunction.class, "applyAsInt")); + methodMap.put(long.class, Tuple2.of(ToLongFunction.class, "applyAsLong")); + methodMap.put(float.class, Tuple2.of(ToFloatFunction.class, "applyAsFloat")); + methodMap.put(double.class, Tuple2.of(ToDoubleFunction.class, "applyAsDouble")); + } + + public static Tuple2, String> getterMethodInfo(Class type) { + Tuple2, String> info = methodMap.get(type); + if (info == null) { + return Tuple2.of(Function.class, "apply"); + } + return info; + } + + public static Object makeGetterFunction( + MethodHandles.Lookup lookup, MethodHandle handle, Class returnType) { + Tuple2, String> methodInfo = methodMap.get(returnType); + MethodType factoryType; + if (methodInfo == null) { + methodInfo = Tuple2.of(Function.class, "apply"); + factoryType = jdkFunctionMethodType; + } else { + factoryType = MethodType.methodType(returnType, Object.class); + } + try { + CallSite callSite = + LambdaMetafactory.metafactory( + lookup, + methodInfo.f1, + MethodType.methodType(methodInfo.f0), + factoryType, + handle, + handle.type()); + return callSite.getTarget().invoke(); + } catch (ClassNotFoundException | NoClassDefFoundError e) { + return makeGetterFunction(lookup, handle, Object.class); + } catch (Throwable e) { + throw ExceptionUtils.throwException(e); + } + } + + public static Object getModule(Class cls) { + Preconditions.checkArgument(JdkVersion.MAJOR_VERSION >= 9); + return cls.getModule(); + } + + public static Object addReads(Object thisModule, Object otherModule) { + Preconditions.checkArgument(JdkVersion.MAJOR_VERSION >= 9); + return ((Module) thisModule).addReads((Module) otherModule); + } +} diff --git a/java/fory-core/src/main/java25/org/apache/fory/platform/internal/_Lookup.java b/java/fory-core/src/main/java25/org/apache/fory/platform/internal/_Lookup.java new file mode 100644 index 0000000000..0d3a669321 --- /dev/null +++ b/java/fory-core/src/main/java25/org/apache/fory/platform/internal/_Lookup.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.platform.internal; + +import java.lang.invoke.MethodHandles; +import java.lang.invoke.MethodHandles.Lookup; + +// CHECKSTYLE.OFF:TypeName +class _Lookup { + // CHECKSTYLE.ON:TypeName + static final Lookup IMPL_LOOKUP = MethodHandles.lookup(); + + // CHECKSTYLE.OFF:MethodName + public static Lookup _trustedLookup(Class objectClass) { + // CHECKSTYLE.ON:MethodName + return privateLookupIn(objectClass, MethodHandles.lookup()); + } + + public static Lookup privateLookupIn(Class targetClass, Lookup caller) { + try { + return MethodHandles.privateLookupIn(targetClass, caller); + } catch (IllegalAccessException e) { + throw new IllegalStateException(privateAccessMessage(targetClass), e); + } + } + + /** + * Creates and links a class or interface from {@code bytes} with the same class loader and in the + * same runtime package and protection domain as this lookup's lookup class. Classes in bytecode + * must be in the same package as the lookup class. + */ + public static Class defineClass(Lookup lookup, byte[] bytes) { + try { + return lookup.defineClass(bytes); + } catch (IllegalAccessException e) { + throw new IllegalStateException(privateAccessMessage(lookup.lookupClass()), e); + } + } + + private static String privateAccessMessage(Class targetClass) { + Module module = targetClass.getModule(); + Package pkg = targetClass.getPackage(); + String packageName = pkg == null ? "" : pkg.getName(); + return "Private lookup for " + + targetClass.getName() + + " requires package " + + packageName + + " in module " + + module.getName() + + " to be open to org.apache.fory.core"; + } +} From 0947efd2cdfc9e77154c34d617926aeb960b5383 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sat, 23 May 2026 23:59:21 +0800 Subject: [PATCH 13/69] feat(java): add JDK25 field accessor --- .../apache/fory/reflect/FieldAccessor.java | 1341 +++++++++++++++++ 1 file changed, 1341 insertions(+) create mode 100644 java/fory-core/src/main/java25/org/apache/fory/reflect/FieldAccessor.java diff --git a/java/fory-core/src/main/java25/org/apache/fory/reflect/FieldAccessor.java b/java/fory-core/src/main/java25/org/apache/fory/reflect/FieldAccessor.java new file mode 100644 index 0000000000..3f0e9a2ff7 --- /dev/null +++ b/java/fory-core/src/main/java25/org/apache/fory/reflect/FieldAccessor.java @@ -0,0 +1,1341 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.reflect; + +import java.lang.invoke.MethodHandle; +import java.lang.invoke.MethodHandles; +import java.lang.invoke.MethodType; +import java.lang.invoke.VarHandle; +import java.lang.reflect.Field; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.lang.reflect.Modifier; +import java.util.function.Function; +import java.util.function.Predicate; +import java.util.function.ToDoubleFunction; +import java.util.function.ToIntFunction; +import java.util.function.ToLongFunction; +import org.apache.fory.exception.ForyException; +import org.apache.fory.platform.AndroidSupport; +import org.apache.fory.platform.GraalvmSupport; +import org.apache.fory.type.TypeUtils; +import org.apache.fory.util.Preconditions; +import org.apache.fory.util.function.ToByteFunction; +import org.apache.fory.util.function.ToCharFunction; +import org.apache.fory.util.function.ToFloatFunction; +import org.apache.fory.util.function.ToShortFunction; +import org.apache.fory.util.record.RecordUtils; + +/** Field accessor for primitive types and object types. */ +@SuppressWarnings({"unchecked", "rawtypes"}) +public abstract class FieldAccessor { + protected final Field field; + protected final long fieldOffset; + + public FieldAccessor(Field field) { + this(field, -1); + } + + protected FieldAccessor(Field field, long fieldOffset) { + this.field = field; + this.fieldOffset = fieldOffset; + Preconditions.checkNotNull(field); + } + + public abstract Object get(Object obj); + + public void set(Object obj, Object value) { + throw new UnsupportedOperationException("Unsupported for field " + field); + } + + public Field getField() { + return field; + } + + public boolean getBoolean(Object targetObject) { + return (Boolean) get(targetObject); + } + + public void putBoolean(Object targetObject, boolean value) { + set(targetObject, value); + } + + public byte getByte(Object targetObject) { + return (Byte) get(targetObject); + } + + public void putByte(Object targetObject, byte value) { + set(targetObject, value); + } + + public char getChar(Object targetObject) { + return (Character) get(targetObject); + } + + public void putChar(Object targetObject, char value) { + set(targetObject, value); + } + + public short getShort(Object targetObject) { + return (Short) get(targetObject); + } + + public void putShort(Object targetObject, short value) { + set(targetObject, value); + } + + public int getInt(Object targetObject) { + return (Integer) get(targetObject); + } + + public void putInt(Object targetObject, int value) { + set(targetObject, value); + } + + public long getLong(Object targetObject) { + return (Long) get(targetObject); + } + + public void putLong(Object targetObject, long value) { + set(targetObject, value); + } + + public float getFloat(Object targetObject) { + return (Float) get(targetObject); + } + + public void putFloat(Object targetObject, float value) { + set(targetObject, value); + } + + public double getDouble(Object targetObject) { + return (Double) get(targetObject); + } + + public void putDouble(Object targetObject, double value) { + set(targetObject, value); + } + + public void putObject(Object targetObject, Object object) { + set(targetObject, object); + } + + public Object getObject(Object targetObject) { + return get(targetObject); + } + + void checkObj(Object obj) { + if (!this.field.getDeclaringClass().isAssignableFrom(obj.getClass())) { + throw new IllegalArgumentException("Illegal class " + obj.getClass()); + } + } + + @Override + public String toString() { + return field.toString(); + } + + public abstract static class FieldGetter extends FieldAccessor { + + private final Object getter; + + protected FieldGetter(Field field, Object getter) { + super(field, -1); + this.getter = getter; + } + + public Object getGetter() { + return getter; + } + } + + public static FieldAccessor createAccessor(Field field) { + Preconditions.checkArgument(!Modifier.isStatic(field.getModifiers()), field); + if (RecordUtils.isRecord(field.getDeclaringClass())) { + if (AndroidSupport.IS_ANDROID || GraalvmSupport.IN_GRAALVM_NATIVE_IMAGE) { + return new ReflectiveRecordFieldAccessor(field); + } + return createRecordAccessor(field); + } + if (AndroidSupport.IS_ANDROID) { + // Android field access must stay reflection-owned: no Unsafe offsets, trusted lookups, + // generated accessors, or primitive-specific reflection subclasses. + return new ReflectionFieldAccessor(field); + } + if (GraalvmSupport.isGraalBuildTime()) { + return new GeneratedAccessor(field); + } + return createVarHandleAccessor(field); + } + + public static FieldAccessor createStaticAccessor(Field field) { + Preconditions.checkArgument(Modifier.isStatic(field.getModifiers()), field); + if (AndroidSupport.IS_ANDROID) { + field.setAccessible(true); + return new ReflectiveStaticFieldAccessor(field); + } + return createVarHandleAccessor(field); + } + + private static FieldAccessor createVarHandleAccessor(Field field) { + if (field.getType() == boolean.class) { + return new BooleanAccessor(field); + } else if (field.getType() == byte.class) { + return new ByteAccessor(field); + } else if (field.getType() == char.class) { + return new CharAccessor(field); + } else if (field.getType() == short.class) { + return new ShortAccessor(field); + } else if (field.getType() == int.class) { + return new IntAccessor(field); + } else if (field.getType() == long.class) { + return new LongAccessor(field); + } else if (field.getType() == float.class) { + return new FloatAccessor(field); + } else if (field.getType() == double.class) { + return new DoubleAccessor(field); + } else { + return new ObjectAccessor(field); + } + } + + private static FieldAccessor createRecordAccessor(Field field) { + MethodHandle getter = recordGetter(field); + if (field.getType() == boolean.class) { + return new BooleanGetter(field, getter); + } else if (field.getType() == byte.class) { + return new ByteGetter(field, getter); + } else if (field.getType() == char.class) { + return new CharGetter(field, getter); + } else if (field.getType() == short.class) { + return new ShortGetter(field, getter); + } else if (field.getType() == int.class) { + return new IntGetter(field, getter); + } else if (field.getType() == long.class) { + return new LongGetter(field, getter); + } else if (field.getType() == float.class) { + return new FloatGetter(field, getter); + } else if (field.getType() == double.class) { + return new DoubleGetter(field, getter); + } else { + return new ObjectGetter(field, getter); + } + } + + private static VarHandle fieldHandle(Field field) { + MethodHandles.Lookup lookup = privateLookup(field); + try { + if (Modifier.isStatic(field.getModifiers())) { + return lookup.findStaticVarHandle( + field.getDeclaringClass(), field.getName(), field.getType()); + } + return lookup.findVarHandle(field.getDeclaringClass(), field.getName(), field.getType()); + } catch (IllegalAccessException e) { + throw accessFailure(field, e); + } catch (NoSuchFieldException e) { + throw new IllegalStateException("Failed to create VarHandle for field " + field, e); + } + } + + private static MethodHandle recordGetter(Field field) { + MethodHandles.Lookup lookup = privateLookup(field); + try { + return lookup.findVirtual( + field.getDeclaringClass(), field.getName(), MethodType.methodType(field.getType())); + } catch (IllegalAccessException e) { + throw accessFailure(field, e); + } catch (NoSuchMethodException e) { + throw new IllegalStateException("Failed to find record accessor for field " + field, e); + } + } + + private static MethodHandles.Lookup privateLookup(Field field) { + Class declaringClass = field.getDeclaringClass(); + try { + return MethodHandles.privateLookupIn(declaringClass, MethodHandles.lookup()); + } catch (IllegalAccessException e) { + throw accessFailure(field, e); + } + } + + private static IllegalStateException accessFailure(Field field, Throwable cause) { + Class declaringClass = field.getDeclaringClass(); + Module targetModule = declaringClass.getModule(); + Package targetPackage = declaringClass.getPackage(); + String packageName = targetPackage == null ? "" : targetPackage.getName(); + return new IllegalStateException( + "Cannot access field " + + field + + " because package " + + packageName + + " in module " + + moduleName(targetModule) + + " is not open to " + + moduleName(FieldAccessor.class.getModule()), + cause); + } + + private static String moduleName(Module module) { + String name = module.getName(); + return name == null ? "" : name; + } + + private static UnsupportedOperationException unsupportedWrite(Field field, Throwable cause) { + return new UnsupportedOperationException( + "Field cannot be written through supported JDK access APIs: " + field, cause); + } + + private static RuntimeException getterFailure(Field field, Throwable cause) { + return new RuntimeException("Failed to read record field: " + field, cause); + } + + private static RuntimeException accessorFailure(Field field, Throwable cause) { + return new RuntimeException("Failed to access field: " + field, cause); + } + + static final class ReflectiveRecordFieldAccessor extends FieldGetter { + private final Method accessor; + + ReflectiveRecordFieldAccessor(Field field) { + super(field, null); + try { + accessor = field.getDeclaringClass().getDeclaredMethod(field.getName()); + accessor.setAccessible(true); + } catch (NoSuchMethodException | RuntimeException e) { + throw new ForyException("Failed to create record field accessor for " + field, e); + } + } + + @Override + public Object get(Object obj) { + checkObj(obj); + try { + return accessor.invoke(obj); + } catch (IllegalAccessException | IllegalArgumentException e) { + throw new ForyException("Failed to read record field reflectively: " + field, e); + } catch (InvocationTargetException e) { + throw new ForyException( + "Record accessor threw while reading field: " + field, e.getCause()); + } + } + + @Override + public void set(Object obj, Object value) { + throw new UnsupportedOperationException("Record field is read-only: " + field); + } + } + + private abstract static class VarHandleAccessor extends FieldAccessor { + protected final VarHandle handle; + protected final boolean isStatic; + + VarHandleAccessor(Field field) { + super(field, -1); + handle = fieldHandle(field); + isStatic = Modifier.isStatic(field.getModifiers()); + } + } + + /** Primitive boolean accessor. */ + public static class BooleanAccessor extends VarHandleAccessor { + public BooleanAccessor(Field field) { + super(field); + Preconditions.checkArgument(field.getType() == boolean.class); + } + + @Override + public Object get(Object obj) { + return getBoolean(obj); + } + + @Override + public boolean getBoolean(Object obj) { + if (isStatic) { + return (boolean) handle.get(); + } + checkObj(obj); + return (boolean) handle.get(obj); + } + + @Override + public void set(Object obj, Object value) { + putBoolean(obj, (Boolean) value); + } + + @Override + public void putBoolean(Object obj, boolean value) { + try { + if (isStatic) { + handle.set(value); + } else { + checkObj(obj); + handle.set(obj, value); + } + } catch (UnsupportedOperationException e) { + throw unsupportedWrite(field, e); + } + } + } + + public static class BooleanGetter extends FieldGetter { + private final Predicate getter; + private final MethodHandle getterHandle; + + public BooleanGetter(Field field, Predicate getter) { + super(field, getter); + this.getter = getter; + getterHandle = null; + Preconditions.checkArgument(field.getType() == boolean.class); + } + + private BooleanGetter(Field field, MethodHandle getter) { + super(field, getter); + this.getter = null; + getterHandle = getter; + Preconditions.checkArgument(field.getType() == boolean.class); + } + + @Override + public Boolean get(Object obj) { + return getBoolean(obj); + } + + @Override + public boolean getBoolean(Object obj) { + checkObj(obj); + if (getterHandle == null) { + return getter.test(obj); + } + try { + return (boolean) getterHandle.invoke(obj); + } catch (Throwable e) { + throw getterFailure(field, e); + } + } + } + + /** Primitive byte accessor. */ + public static class ByteAccessor extends VarHandleAccessor { + public ByteAccessor(Field field) { + super(field); + Preconditions.checkArgument(field.getType() == byte.class); + } + + @Override + public Byte get(Object obj) { + return getByte(obj); + } + + @Override + public byte getByte(Object obj) { + if (isStatic) { + return (byte) handle.get(); + } + checkObj(obj); + return (byte) handle.get(obj); + } + + @Override + public void set(Object obj, Object value) { + putByte(obj, (Byte) value); + } + + @Override + public void putByte(Object obj, byte value) { + try { + if (isStatic) { + handle.set(value); + } else { + checkObj(obj); + handle.set(obj, value); + } + } catch (UnsupportedOperationException e) { + throw unsupportedWrite(field, e); + } + } + } + + public static class ByteGetter extends FieldGetter { + + private final ToByteFunction getter; + private final MethodHandle getterHandle; + + public ByteGetter(Field field, ToByteFunction getter) { + super(field, getter); + this.getter = getter; + getterHandle = null; + Preconditions.checkArgument(field.getType() == byte.class); + } + + private ByteGetter(Field field, MethodHandle getter) { + super(field, getter); + this.getter = null; + getterHandle = getter; + Preconditions.checkArgument(field.getType() == byte.class); + } + + @Override + public Byte get(Object obj) { + return getByte(obj); + } + + @Override + public byte getByte(Object obj) { + checkObj(obj); + if (getterHandle == null) { + return getter.applyAsByte(obj); + } + try { + return (byte) getterHandle.invoke(obj); + } catch (Throwable e) { + throw getterFailure(field, e); + } + } + } + + /** Primitive char accessor. */ + public static class CharAccessor extends VarHandleAccessor { + public CharAccessor(Field field) { + super(field); + Preconditions.checkArgument(field.getType() == char.class); + } + + @Override + public Character get(Object obj) { + return getChar(obj); + } + + @Override + public char getChar(Object obj) { + if (isStatic) { + return (char) handle.get(); + } + checkObj(obj); + return (char) handle.get(obj); + } + + @Override + public void set(Object obj, Object value) { + putChar(obj, (Character) value); + } + + @Override + public void putChar(Object obj, char value) { + try { + if (isStatic) { + handle.set(value); + } else { + checkObj(obj); + handle.set(obj, value); + } + } catch (UnsupportedOperationException e) { + throw unsupportedWrite(field, e); + } + } + } + + public static class CharGetter extends FieldGetter { + private final ToCharFunction getter; + private final MethodHandle getterHandle; + + public CharGetter(Field field, ToCharFunction getter) { + super(field, getter); + this.getter = getter; + getterHandle = null; + Preconditions.checkArgument(field.getType() == char.class); + } + + private CharGetter(Field field, MethodHandle getter) { + super(field, getter); + this.getter = null; + getterHandle = getter; + Preconditions.checkArgument(field.getType() == char.class); + } + + @Override + public Character get(Object obj) { + return getChar(obj); + } + + @Override + public char getChar(Object obj) { + checkObj(obj); + if (getterHandle == null) { + return getter.applyAsChar(obj); + } + try { + return (char) getterHandle.invoke(obj); + } catch (Throwable e) { + throw getterFailure(field, e); + } + } + } + + /** Primitive short accessor. */ + public static class ShortAccessor extends VarHandleAccessor { + public ShortAccessor(Field field) { + super(field); + Preconditions.checkArgument(field.getType() == short.class); + } + + @Override + public Short get(Object obj) { + return getShort(obj); + } + + @Override + public short getShort(Object obj) { + if (isStatic) { + return (short) handle.get(); + } + checkObj(obj); + return (short) handle.get(obj); + } + + @Override + public void set(Object obj, Object value) { + putShort(obj, (Short) value); + } + + @Override + public void putShort(Object obj, short value) { + try { + if (isStatic) { + handle.set(value); + } else { + checkObj(obj); + handle.set(obj, value); + } + } catch (UnsupportedOperationException e) { + throw unsupportedWrite(field, e); + } + } + } + + public static class ShortGetter extends FieldGetter { + private final ToShortFunction getter; + private final MethodHandle getterHandle; + + public ShortGetter(Field field, ToShortFunction getter) { + super(field, getter); + this.getter = getter; + getterHandle = null; + Preconditions.checkArgument(field.getType() == short.class); + } + + private ShortGetter(Field field, MethodHandle getter) { + super(field, getter); + this.getter = null; + getterHandle = getter; + Preconditions.checkArgument(field.getType() == short.class); + } + + @Override + public Short get(Object obj) { + return getShort(obj); + } + + @Override + public short getShort(Object obj) { + checkObj(obj); + if (getterHandle == null) { + return getter.applyAsShort(obj); + } + try { + return (short) getterHandle.invoke(obj); + } catch (Throwable e) { + throw getterFailure(field, e); + } + } + } + + /** Primitive int accessor. */ + public static class IntAccessor extends VarHandleAccessor { + public IntAccessor(Field field) { + super(field); + Preconditions.checkArgument(field.getType() == int.class); + } + + @Override + public Integer get(Object obj) { + return getInt(obj); + } + + @Override + public int getInt(Object obj) { + if (isStatic) { + return (int) handle.get(); + } + checkObj(obj); + return (int) handle.get(obj); + } + + @Override + public void set(Object obj, Object value) { + putInt(obj, (Integer) value); + } + + @Override + public void putInt(Object obj, int value) { + try { + if (isStatic) { + handle.set(value); + } else { + checkObj(obj); + handle.set(obj, value); + } + } catch (UnsupportedOperationException e) { + throw unsupportedWrite(field, e); + } + } + } + + public static class IntGetter extends FieldGetter { + private final ToIntFunction getter; + private final MethodHandle getterHandle; + + public IntGetter(Field field, ToIntFunction getter) { + super(field, getter); + this.getter = getter; + getterHandle = null; + Preconditions.checkArgument(field.getType() == int.class); + } + + private IntGetter(Field field, MethodHandle getter) { + super(field, getter); + this.getter = null; + getterHandle = getter; + Preconditions.checkArgument(field.getType() == int.class); + } + + @Override + public Integer get(Object obj) { + return getInt(obj); + } + + @Override + public int getInt(Object obj) { + checkObj(obj); + if (getterHandle == null) { + return getter.applyAsInt(obj); + } + try { + return (int) getterHandle.invoke(obj); + } catch (Throwable e) { + throw getterFailure(field, e); + } + } + } + + /** Primitive long accessor. */ + public static class LongAccessor extends VarHandleAccessor { + public LongAccessor(Field field) { + super(field); + Preconditions.checkArgument(field.getType() == long.class); + } + + @Override + public Long get(Object obj) { + return getLong(obj); + } + + @Override + public long getLong(Object obj) { + if (isStatic) { + return (long) handle.get(); + } + checkObj(obj); + return (long) handle.get(obj); + } + + @Override + public void set(Object obj, Object value) { + putLong(obj, (Long) value); + } + + @Override + public void putLong(Object obj, long value) { + try { + if (isStatic) { + handle.set(value); + } else { + checkObj(obj); + handle.set(obj, value); + } + } catch (UnsupportedOperationException e) { + throw unsupportedWrite(field, e); + } + } + } + + public static class LongGetter extends FieldGetter { + private final ToLongFunction getter; + private final MethodHandle getterHandle; + + public LongGetter(Field field, ToLongFunction getter) { + super(field, getter); + this.getter = getter; + getterHandle = null; + Preconditions.checkArgument(field.getType() == long.class); + } + + private LongGetter(Field field, MethodHandle getter) { + super(field, getter); + this.getter = null; + getterHandle = getter; + Preconditions.checkArgument(field.getType() == long.class); + } + + @Override + public Long get(Object obj) { + return getLong(obj); + } + + @Override + public long getLong(Object obj) { + checkObj(obj); + if (getterHandle == null) { + return getter.applyAsLong(obj); + } + try { + return (long) getterHandle.invoke(obj); + } catch (Throwable e) { + throw getterFailure(field, e); + } + } + } + + /** Primitive float accessor. */ + public static class FloatAccessor extends VarHandleAccessor { + public FloatAccessor(Field field) { + super(field); + Preconditions.checkArgument(field.getType() == float.class); + } + + @Override + public Object get(Object obj) { + return getFloat(obj); + } + + @Override + public float getFloat(Object obj) { + if (isStatic) { + return (float) handle.get(); + } + checkObj(obj); + return (float) handle.get(obj); + } + + @Override + public void set(Object obj, Object value) { + putFloat(obj, (Float) value); + } + + @Override + public void putFloat(Object obj, float value) { + try { + if (isStatic) { + handle.set(value); + } else { + checkObj(obj); + handle.set(obj, value); + } + } catch (UnsupportedOperationException e) { + throw unsupportedWrite(field, e); + } + } + } + + public static class FloatGetter extends FieldGetter { + private final ToFloatFunction getter; + private final MethodHandle getterHandle; + + public FloatGetter(Field field, ToFloatFunction getter) { + super(field, getter); + this.getter = getter; + getterHandle = null; + Preconditions.checkArgument(field.getType() == float.class); + } + + private FloatGetter(Field field, MethodHandle getter) { + super(field, getter); + this.getter = null; + getterHandle = getter; + Preconditions.checkArgument(field.getType() == float.class); + } + + @Override + public Float get(Object obj) { + return getFloat(obj); + } + + @Override + public float getFloat(Object obj) { + checkObj(obj); + if (getterHandle == null) { + return getter.applyAsFloat(obj); + } + try { + return (float) getterHandle.invoke(obj); + } catch (Throwable e) { + throw getterFailure(field, e); + } + } + } + + /** Primitive double accessor. */ + public static class DoubleAccessor extends VarHandleAccessor { + public DoubleAccessor(Field field) { + super(field); + Preconditions.checkArgument(field.getType() == double.class); + } + + @Override + public Object get(Object obj) { + return getDouble(obj); + } + + @Override + public double getDouble(Object obj) { + if (isStatic) { + return (double) handle.get(); + } + checkObj(obj); + return (double) handle.get(obj); + } + + @Override + public void set(Object obj, Object value) { + putDouble(obj, (Double) value); + } + + @Override + public void putDouble(Object obj, double value) { + try { + if (isStatic) { + handle.set(value); + } else { + checkObj(obj); + handle.set(obj, value); + } + } catch (UnsupportedOperationException e) { + throw unsupportedWrite(field, e); + } + } + } + + public static class DoubleGetter extends FieldGetter { + private final ToDoubleFunction getter; + private final MethodHandle getterHandle; + + public DoubleGetter(Field field, ToDoubleFunction getter) { + super(field, getter); + this.getter = getter; + getterHandle = null; + Preconditions.checkArgument(field.getType() == double.class); + } + + private DoubleGetter(Field field, MethodHandle getter) { + super(field, getter); + this.getter = null; + getterHandle = getter; + Preconditions.checkArgument(field.getType() == double.class); + } + + @Override + public Double get(Object obj) { + return getDouble(obj); + } + + @Override + public double getDouble(Object obj) { + checkObj(obj); + if (getterHandle == null) { + return getter.applyAsDouble(obj); + } + try { + return (double) getterHandle.invoke(obj); + } catch (Throwable e) { + throw getterFailure(field, e); + } + } + } + + /** Object accessor. */ + public static class ObjectAccessor extends VarHandleAccessor { + public ObjectAccessor(Field field) { + super(field); + Preconditions.checkArgument(!TypeUtils.isPrimitive(field.getType())); + } + + @Override + public Object get(Object obj) { + if (isStatic) { + return handle.get(); + } + checkObj(obj); + return handle.get(obj); + } + + @Override + public void set(Object obj, Object value) { + try { + if (isStatic) { + handle.set(value); + } else { + checkObj(obj); + handle.set(obj, value); + } + } catch (UnsupportedOperationException e) { + throw unsupportedWrite(field, e); + } + } + } + + public static class ObjectGetter extends FieldGetter { + private final Function getter; + private final MethodHandle getterHandle; + + public ObjectGetter(Field field, Function getter) { + super(field, getter); + this.getter = getter; + getterHandle = null; + Preconditions.checkArgument(!field.getType().isPrimitive(), field); + } + + private ObjectGetter(Field field, MethodHandle getter) { + super(field, getter); + this.getter = null; + getterHandle = getter; + Preconditions.checkArgument(!field.getType().isPrimitive(), field); + } + + @Override + public Object get(Object obj) { + checkObj(obj); + if (getterHandle == null) { + return getter.apply(obj); + } + try { + return getterHandle.invoke(obj); + } catch (Throwable e) { + throw getterFailure(field, e); + } + } + } + + static final class ReflectiveStaticFieldAccessor extends FieldAccessor { + ReflectiveStaticFieldAccessor(Field field) { + super(field, -1); + } + + @Override + public Object get(Object obj) { + try { + return field.get(null); + } catch (IllegalAccessException | IllegalArgumentException e) { + throw new ForyException("Failed to read static field reflectively: " + field, e); + } + } + + @Override + public void set(Object obj, Object value) { + try { + field.set(null, value); + } catch (IllegalAccessException | IllegalArgumentException e) { + throw new ForyException("Failed to write static field reflectively: " + field, e); + } + } + } + + static final class StaticObjectAccessor extends ObjectAccessor { + StaticObjectAccessor(Field field) { + super(field); + Preconditions.checkArgument(Modifier.isStatic(field.getModifiers()), field); + } + } + + static final class GeneratedAccessor extends VarHandleAccessor { + GeneratedAccessor(Field field) { + super(field); + } + + @Override + public Object get(Object obj) { + try { + if (isStatic) { + return handle.get(); + } + checkObj(obj); + return handle.get(obj); + } catch (RuntimeException e) { + throw accessorFailure(field, e); + } + } + + @Override + public void set(Object obj, Object value) { + try { + if (isStatic) { + handle.set(value); + } else { + checkObj(obj); + handle.set(obj, value); + } + } catch (UnsupportedOperationException e) { + throw unsupportedWrite(field, e); + } catch (RuntimeException e) { + throw accessorFailure(field, e); + } + } + + @Override + public boolean getBoolean(Object obj) { + try { + if (isStatic) { + return (boolean) handle.get(); + } + checkObj(obj); + return (boolean) handle.get(obj); + } catch (RuntimeException e) { + throw accessorFailure(field, e); + } + } + + @Override + public void putBoolean(Object obj, boolean value) { + try { + if (isStatic) { + handle.set(value); + } else { + checkObj(obj); + handle.set(obj, value); + } + } catch (UnsupportedOperationException e) { + throw unsupportedWrite(field, e); + } catch (RuntimeException e) { + throw accessorFailure(field, e); + } + } + + @Override + public byte getByte(Object obj) { + try { + if (isStatic) { + return (byte) handle.get(); + } + checkObj(obj); + return (byte) handle.get(obj); + } catch (RuntimeException e) { + throw accessorFailure(field, e); + } + } + + @Override + public void putByte(Object obj, byte value) { + try { + if (isStatic) { + handle.set(value); + } else { + checkObj(obj); + handle.set(obj, value); + } + } catch (UnsupportedOperationException e) { + throw unsupportedWrite(field, e); + } catch (RuntimeException e) { + throw accessorFailure(field, e); + } + } + + @Override + public char getChar(Object obj) { + try { + if (isStatic) { + return (char) handle.get(); + } + checkObj(obj); + return (char) handle.get(obj); + } catch (RuntimeException e) { + throw accessorFailure(field, e); + } + } + + @Override + public void putChar(Object obj, char value) { + try { + if (isStatic) { + handle.set(value); + } else { + checkObj(obj); + handle.set(obj, value); + } + } catch (UnsupportedOperationException e) { + throw unsupportedWrite(field, e); + } catch (RuntimeException e) { + throw accessorFailure(field, e); + } + } + + @Override + public short getShort(Object obj) { + try { + if (isStatic) { + return (short) handle.get(); + } + checkObj(obj); + return (short) handle.get(obj); + } catch (RuntimeException e) { + throw accessorFailure(field, e); + } + } + + @Override + public void putShort(Object obj, short value) { + try { + if (isStatic) { + handle.set(value); + } else { + checkObj(obj); + handle.set(obj, value); + } + } catch (UnsupportedOperationException e) { + throw unsupportedWrite(field, e); + } catch (RuntimeException e) { + throw accessorFailure(field, e); + } + } + + @Override + public int getInt(Object obj) { + try { + if (isStatic) { + return (int) handle.get(); + } + checkObj(obj); + return (int) handle.get(obj); + } catch (RuntimeException e) { + throw accessorFailure(field, e); + } + } + + @Override + public void putInt(Object obj, int value) { + try { + if (isStatic) { + handle.set(value); + } else { + checkObj(obj); + handle.set(obj, value); + } + } catch (UnsupportedOperationException e) { + throw unsupportedWrite(field, e); + } catch (RuntimeException e) { + throw accessorFailure(field, e); + } + } + + @Override + public long getLong(Object obj) { + try { + if (isStatic) { + return (long) handle.get(); + } + checkObj(obj); + return (long) handle.get(obj); + } catch (RuntimeException e) { + throw accessorFailure(field, e); + } + } + + @Override + public void putLong(Object obj, long value) { + try { + if (isStatic) { + handle.set(value); + } else { + checkObj(obj); + handle.set(obj, value); + } + } catch (UnsupportedOperationException e) { + throw unsupportedWrite(field, e); + } catch (RuntimeException e) { + throw accessorFailure(field, e); + } + } + + @Override + public float getFloat(Object obj) { + try { + if (isStatic) { + return (float) handle.get(); + } + checkObj(obj); + return (float) handle.get(obj); + } catch (RuntimeException e) { + throw accessorFailure(field, e); + } + } + + @Override + public void putFloat(Object obj, float value) { + try { + if (isStatic) { + handle.set(value); + } else { + checkObj(obj); + handle.set(obj, value); + } + } catch (UnsupportedOperationException e) { + throw unsupportedWrite(field, e); + } catch (RuntimeException e) { + throw accessorFailure(field, e); + } + } + + @Override + public double getDouble(Object obj) { + try { + if (isStatic) { + return (double) handle.get(); + } + checkObj(obj); + return (double) handle.get(obj); + } catch (RuntimeException e) { + throw accessorFailure(field, e); + } + } + + @Override + public void putDouble(Object obj, double value) { + try { + if (isStatic) { + handle.set(value); + } else { + checkObj(obj); + handle.set(obj, value); + } + } catch (UnsupportedOperationException e) { + throw unsupportedWrite(field, e); + } catch (RuntimeException e) { + throw accessorFailure(field, e); + } + } + } +} From 1ed30ddb0cd107f62c0ee5e497cf8b8da69ff25b Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sun, 24 May 2026 14:16:03 +0800 Subject: [PATCH 14/69] feat(java): add JDK25 zero-unsafe runtime path --- .github/workflows/release-java-snapshot.yaml | 6 +- benchmarks/java/pom.xml | 152 + .../fory/benchmark/Jdk25MrJarCheck.java | 50 + .../fory/benchmark/NewJava11StringSuite.java | 20 +- .../apache/fory/benchmark/NewStringSuite.java | 12 +- ci/deploy.sh | 11 + ci/run_ci.sh | 114 +- ci/tasks/java.py | 135 +- docs/guide/java/troubleshooting.md | 44 + integration_tests/graalvm_tests/pom.xml | 33 + .../fory/graalvm/FeatureTestExample.java | 2 + .../java/org/apache/fory/graalvm/Foo.java | 2 + .../fory/graalvm/ObjectStreamExample.java | 12 +- .../jdk_compatibility_tests/.gitignore | 4 + .../jdk_compatibility_tests/pom.xml | 33 + integration_tests/jpms_tests/pom.xml | 33 + .../jpms_tests/src/main/java/module-info.java | 4 + .../model/PrivateFieldBean.java | 32 + .../PublicSerializerValue.java | 28 + .../PublicSerializerValueSerializer.java | 41 + .../JpmsFieldAccessorTest.java | 50 + java/fory-core/pom.xml | 260 + .../src/main/java/org/apache/fory/Fory.java | 2 +- .../org/apache/fory/builder/CodecBuilder.java | 10 +- .../fory/builder/CompatibleCodecBuilder.java | 91 +- .../fory/builder/ObjectCodecBuilder.java | 380 +- .../org/apache/fory/collection/MapEntry.java | 2 + .../org/apache/fory/context/CopyContext.java | 23 +- .../org/apache/fory/context/MapRefReader.java | 86 +- .../org/apache/fory/context/ReadContext.java | 30 + .../org/apache/fory/context/RefReader.java | 29 + .../org/apache/fory/memory/MemoryUtils.java | 22 +- .../java/org/apache/fory/meta/TypeDef.java | 3 - .../org/apache/fory/platform/UnsafeOps.java | 6 + .../fory/platform/internal/DefineClass.java | 5 + .../fory/platform/internal/_JDKAccess.java | 216 +- .../apache/fory/reflect/FieldAccessor.java | 108 +- .../apache/fory/reflect/ObjectCreator.java | 32 + .../apache/fory/reflect/ObjectCreators.java | 442 +- .../apache/fory/reflect/ReflectionUtils.java | 31 - .../apache/fory/resolver/ClassResolver.java | 73 +- .../serializer/AbstractObjectSerializer.java | 361 +- .../CompatibleLayerSerializerBase.java | 25 +- .../fory/serializer/CompatibleSerializer.java | 234 +- .../fory/serializer/ExceptionSerializers.java | 64 +- .../FinalFieldReplaceResolveSerializer.java | 4 +- .../fory/serializer/JavaSerializer.java | 33 +- .../fory/serializer/JdkProxySerializer.java | 4 +- .../fory/serializer/ObjectSerializer.java | 141 +- .../serializer/ObjectStreamSerializer.java | 109 +- .../serializer/ReplaceResolveSerializer.java | 92 +- .../apache/fory/serializer/Serializers.java | 89 + .../fory/serializer/StringSerializer.java | 157 +- .../apache/fory/serializer/URLSerializer.java | 15 +- .../collection/ChildContainerSerializers.java | 24 +- .../collection/CollectionSerializers.java | 60 +- .../GuavaCollectionSerializers.java | 206 +- .../ImmutableCollectionSerializers.java | 18 +- .../serializer/collection/MapSerializers.java | 5 +- .../collection/SubListSerializers.java | 2 +- .../collection/SynchronizedSerializers.java | 12 +- .../collection/UnmodifiableSerializers.java | 12 +- .../java/org/apache/fory/type/BFloat16.java | 2 + .../org/apache/fory/type/BFloat16Array.java | 6 + .../java/org/apache/fory/type/Float16.java | 2 + .../org/apache/fory/type/Float16Array.java | 6 + .../org/apache/fory/type/unsigned/UInt16.java | 2 + .../org/apache/fory/type/unsigned/UInt32.java | 2 + .../org/apache/fory/type/unsigned/UInt64.java | 2 + .../org/apache/fory/type/unsigned/UInt8.java | 2 + .../apache/fory/util/DefaultValueUtils.java | 108 +- .../org/apache/fory/builder/CodecBuilder.java | 732 +++ .../fory/builder/ObjectCodecBuilder.java | 1465 ++++++ .../org/apache/fory/memory/MemoryBuffer.java | 4521 +++++++++++++++++ .../org/apache/fory/platform/UnsafeOps.java | 80 +- .../fory/platform/internal/DefineClass.java | 110 + .../fory/platform/internal/_JDKAccess.java | 519 +- .../fory/platform/internal/_Lookup.java | 7 +- .../apache/fory/reflect/FieldAccessor.java | 267 +- .../reflect/HiddenFieldAccessorFactory.java | 559 ++ .../SerializedLambdaSerializer.java | 216 + .../apache/fory/serializer/Serializers.java | 865 ++++ .../fory/serializer/SlicedStringUtil.java | 285 ++ .../fory/serializer/StringSerializer.java | 1166 +++++ .../fory-core/native-image.properties | 13 +- .../test/java/org/apache/fory/ForyTest.java | 55 +- .../pkgprivate/PackagePrivateMapKeyTest.java | 23 +- .../apache/fory/memory/MemoryBufferTest.java | 64 +- .../org/apache/fory/meta/TypeDefTest.java | 3 - .../fory/reflect/FieldAccessorTest.java | 35 + .../fory/resolver/DisallowedListTest.java | 6 +- .../fory/serializer/ArraySerializersTest.java | 8 + .../fory/serializer/DuplicateFieldsTest.java | 80 + ...inalFieldReplaceResolveSerializerTest.java | 2 - .../fory/serializer/ObjectSerializerTest.java | 463 ++ .../ObjectStreamSerializerTest.java | 2 +- .../ReplaceResolveSerializerTest.java | 2 + .../fory/serializer/URLSerializerTest.java | 2 +- .../collection/CollectionSerializersTest.java | 19 +- .../collection/MapSerializersTest.java | 46 +- java/fory-format/pom.xml | 15 +- .../fory/format/vectorized/ArrowUtils.java | 80 +- .../format/vectorized/ArrowTestSupport.java | 35 + .../format/vectorized/ArrowUtilsTest.java | 1 + .../format/vectorized/ArrowWriterTest.java | 2 + .../ImmutableCollectionSerializersTest.java | 8 +- .../apache/fory/test/bean/AccessBeans.java | 38 +- .../GuavaCollectionSerializersTest.java | 2 + .../org/apache/fory/test/FastJsonTest.java | 2 + .../fory/test/ReadResolveCircularTest.java | 9 +- java/pom.xml | 32 + 111 files changed, 15715 insertions(+), 592 deletions(-) create mode 100644 benchmarks/java/src/main/java/org/apache/fory/benchmark/Jdk25MrJarCheck.java create mode 100644 integration_tests/jdk_compatibility_tests/.gitignore create mode 100644 integration_tests/jpms_tests/src/main/java/org/apache/fory/integration_tests/model/PrivateFieldBean.java create mode 100644 integration_tests/jpms_tests/src/main/java/org/apache/fory/integration_tests/publicserializer/PublicSerializerValue.java create mode 100644 integration_tests/jpms_tests/src/main/java/org/apache/fory/integration_tests/publicserializer/PublicSerializerValueSerializer.java create mode 100644 integration_tests/jpms_tests/src/test/java/org/apache/fory/integration_tests/JpmsFieldAccessorTest.java create mode 100644 java/fory-core/src/main/java25/org/apache/fory/builder/CodecBuilder.java create mode 100644 java/fory-core/src/main/java25/org/apache/fory/builder/ObjectCodecBuilder.java create mode 100644 java/fory-core/src/main/java25/org/apache/fory/memory/MemoryBuffer.java create mode 100644 java/fory-core/src/main/java25/org/apache/fory/platform/internal/DefineClass.java create mode 100644 java/fory-core/src/main/java25/org/apache/fory/reflect/HiddenFieldAccessorFactory.java create mode 100644 java/fory-core/src/main/java25/org/apache/fory/serializer/SerializedLambdaSerializer.java create mode 100644 java/fory-core/src/main/java25/org/apache/fory/serializer/Serializers.java create mode 100644 java/fory-core/src/main/java25/org/apache/fory/serializer/SlicedStringUtil.java create mode 100644 java/fory-core/src/main/java25/org/apache/fory/serializer/StringSerializer.java create mode 100644 java/fory-format/src/test/java/org/apache/fory/format/vectorized/ArrowTestSupport.java diff --git a/.github/workflows/release-java-snapshot.yaml b/.github/workflows/release-java-snapshot.yaml index 631975aac4..af869e9ebf 100644 --- a/.github/workflows/release-java-snapshot.yaml +++ b/.github/workflows/release-java-snapshot.yaml @@ -32,15 +32,15 @@ jobs: - name: Set up Maven Central Repository uses: actions/setup-java@v4 with: - java-version: "11" - distribution: "adopt" + java-version: "25" + distribution: "temurin" architecture: x64 cache: maven server-id: apache.snapshots.https server-username: NEXUS_USERNAME server-password: NEXUS_PASSWORD - name: Publish Fory Java Snapshot - run: python ./ci/run_ci.py java --version 11 --release + run: python ./ci/run_ci.py java --version 25 --release env: NEXUS_USERNAME: ${{ secrets.NEXUS_USER }} NEXUS_PASSWORD: ${{ secrets.NEXUS_PW }} diff --git a/benchmarks/java/pom.xml b/benchmarks/java/pom.xml index bbd9149b6d..07b793301b 100644 --- a/benchmarks/java/pom.xml +++ b/benchmarks/java/pom.xml @@ -236,6 +236,141 @@ + + jdk25-benchmark-mrjar-check + + [25,) + + + + + org.apache.maven.plugins + maven-antrun-plugin + + + verify-benchmark-mrjar + package + + run + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -266,6 +401,7 @@ + true org.apache.fory.benchmark @@ -287,6 +423,10 @@ org.openjdk.jmh.Main + + true + org.apache.fory.benchmark + @@ -298,6 +438,18 @@ META-INF/*.RSA + + org.apache.logging.log4j:* + + META-INF/versions/** + + + + com.fasterxml.jackson.core:jackson-core + + META-INF/versions/** + + diff --git a/benchmarks/java/src/main/java/org/apache/fory/benchmark/Jdk25MrJarCheck.java b/benchmarks/java/src/main/java/org/apache/fory/benchmark/Jdk25MrJarCheck.java new file mode 100644 index 0000000000..857b668e04 --- /dev/null +++ b/benchmarks/java/src/main/java/org/apache/fory/benchmark/Jdk25MrJarCheck.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.benchmark; + +import org.apache.fory.memory.MemoryBuffer; +import org.apache.fory.platform.UnsafeOps; +import org.apache.fory.platform.internal._JDKAccess; +import org.apache.fory.reflect.FieldAccessor; +import org.apache.fory.serializer.StringSerializer; + +/** Runtime smoke check that JDK25 benchmark runs load the multi-release Fory classes. */ +public final class Jdk25MrJarCheck { + private Jdk25MrJarCheck() {} + + public static void main(String[] args) { + verifyClass(MemoryBuffer.class); + verifyClass(UnsafeOps.class); + verifyClass(_JDKAccess.class); + verifyClass(FieldAccessor.class); + verifyClass(StringSerializer.class); + if (_JDKAccess.UNSAFE != null) { + throw new IllegalStateException("JDK25 benchmark jar loaded Unsafe-backed _JDKAccess"); + } + } + + private static void verifyClass(Class cls) { + String resourceName = cls.getSimpleName() + ".class"; + String resource = String.valueOf(cls.getResource(resourceName)); + if (!resource.contains("benchmarks.jar!") || !resource.contains("!/META-INF/versions/25/")) { + throw new IllegalStateException("JDK25 benchmark jar loaded root class for " + cls); + } + } +} diff --git a/benchmarks/java/src/main/java/org/apache/fory/benchmark/NewJava11StringSuite.java b/benchmarks/java/src/main/java/org/apache/fory/benchmark/NewJava11StringSuite.java index 38a380a6dd..c4fcde43e1 100644 --- a/benchmarks/java/src/main/java/org/apache/fory/benchmark/NewJava11StringSuite.java +++ b/benchmarks/java/src/main/java/org/apache/fory/benchmark/NewJava11StringSuite.java @@ -23,7 +23,6 @@ import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.platform.JdkVersion; import org.apache.fory.platform.UnsafeOps; -import org.apache.fory.reflect.ReflectionUtils; import org.apache.fory.serializer.StringSerializer; import org.apache.fory.util.Preconditions; import org.apache.fory.util.StringUtils; @@ -37,16 +36,13 @@ public class NewJava11StringSuite { static { if (JdkVersion.MAJOR_VERSION > 8) { - strBytes = - (byte[]) UnsafeOps.getObject(str, ReflectionUtils.getFieldOffset(String.class, "value")); - coder = UnsafeOps.getByte(str, ReflectionUtils.getFieldOffset(String.class, "coder")); + strBytes = (byte[]) UnsafeOps.getObject(str, fieldOffset(String.class, "value")); + coder = UnsafeOps.getByte(str, fieldOffset(String.class, "coder")); } } - private static final long STRING_VALUE_FIELD_OFFSET = - ReflectionUtils.getFieldOffset(String.class, "value"); - private static final long STRING_CODER_FIELD_OFFSET = - ReflectionUtils.getFieldOffset(String.class, "coder"); + private static final long STRING_VALUE_FIELD_OFFSET = fieldOffset(String.class, "value"); + private static final long STRING_CODER_FIELD_OFFSET = fieldOffset(String.class, "coder"); private static String stubStr = new String(new char[] {Character.MAX_VALUE, Character.MIN_VALUE}); private static Fory fory = @@ -58,6 +54,14 @@ public class NewJava11StringSuite { stringSerializer.writeString(buffer, str); } + private static long fieldOffset(Class type, String fieldName) { + try { + return UnsafeOps.objectFieldOffset(type.getDeclaredField(fieldName)); + } catch (NoSuchFieldException e) { + throw new IllegalStateException(e); + } + } + // @Benchmark public Object createJDK11StringByCopyStr() { return new String(str); diff --git a/benchmarks/java/src/main/java/org/apache/fory/benchmark/NewStringSuite.java b/benchmarks/java/src/main/java/org/apache/fory/benchmark/NewStringSuite.java index 114785ef17..dfbad5136a 100644 --- a/benchmarks/java/src/main/java/org/apache/fory/benchmark/NewStringSuite.java +++ b/benchmarks/java/src/main/java/org/apache/fory/benchmark/NewStringSuite.java @@ -20,7 +20,6 @@ package org.apache.fory.benchmark; import org.apache.fory.platform.UnsafeOps; -import org.apache.fory.reflect.ReflectionUtils; import org.apache.fory.serializer.StringSerializer; import org.apache.fory.util.StringUtils; import org.openjdk.jmh.Main; @@ -41,10 +40,17 @@ public Object createJDK8StringByCopy() { return new String(strData); } - private static final long STRING_VALUE_FIELD_OFFSET = - ReflectionUtils.getFieldOffset(String.class, "value"); + private static final long STRING_VALUE_FIELD_OFFSET = fieldOffset(String.class, "value"); private static String stubStr = new String(new char[] {Character.MAX_VALUE, Character.MIN_VALUE}); + private static long fieldOffset(Class type, String fieldName) { + try { + return UnsafeOps.objectFieldOffset(type.getDeclaredField(fieldName)); + } catch (NoSuchFieldException e) { + throw new IllegalStateException(e); + } + } + // @Benchmark public Object createJDK8StringByUnsafe() { String str = new String(stubStr); diff --git a/ci/deploy.sh b/ci/deploy.sh index fb0123eb16..c88ac210cd 100755 --- a/ci/deploy.sh +++ b/ci/deploy.sh @@ -75,6 +75,17 @@ bump_javascript_version() { } deploy_jars() { + local java_version java_major + java_version=$(java -version 2>&1 | awk -F '"' '/version/ {print $2; exit}') + if [[ "$java_version" == 1.* ]]; then + java_major=$(echo "$java_version" | cut -d. -f2) + else + java_major=$(echo "$java_version" | cut -d. -f1) + fi + if [[ "$java_major" -lt 25 ]]; then + echo "Java releases must run on JDK25+ so MR-JAR entries are packaged" + exit 1 + fi cd "$ROOT/java" mvn -T10 clean deploy --no-transfer-progress -DskipTests -Prelease } diff --git a/ci/run_ci.sh b/ci/run_ci.sh index 45e096a45e..dfe1611726 100755 --- a/ci/run_ci.sh +++ b/ci/run_ci.sh @@ -61,6 +61,7 @@ install_pyfory() { } JDKS=( +"zulu25.30.17-ca-jdk25.0.1-linux_x64" "zulu21.28.85-ca-jdk21.0.0-linux_x64" "zulu17.44.17-ca-crac-jdk17.0.8-linux_x64" "zulu15.46.17-ca-jdk15.0.10-linux_x64" @@ -78,6 +79,17 @@ install_jdks() { } graalvm_test() { + java_version=$(java -version 2>&1 | awk -F '"' '/version/ {print $2; exit}') + if [[ "$java_version" == 1.* ]]; then + java_major=$(echo "$java_version" | cut -d. -f2) + else + java_major=$(echo "$java_version" | cut -d. -f1) + fi + if [[ "$java_major" -ge 25 ]]; then + export JDK_JAVA_OPTIONS="$(jdk25_deny_options) $(jdk25_javac_options)" + else + unset JDK_JAVA_OPTIONS + fi cd "$ROOT"/java mvn -T10 -B --no-transfer-progress clean install -DskipTests -pl '!:fory-testsuite' echo "Start to build graalvm native image" @@ -89,27 +101,97 @@ graalvm_test() { echo "Execute graalvm tests succeed!" } -integration_tests() { +jdk25_deny_options() { + local fory_open_targets="ALL-UNNAMED,org.apache.fory.core,org.apache.fory.format" + printf "%s" "--sun-misc-unsafe-memory-access=deny" + printf " %s" "--add-opens=java.base/java.lang=${fory_open_targets}" + printf " %s" "--add-opens=java.base/java.lang.invoke=${fory_open_targets}" + printf " %s" "--add-opens=java.base/java.lang.reflect=${fory_open_targets}" + printf " %s" "--add-opens=java.base/jdk.internal.reflect=${fory_open_targets}" + printf " %s" "--add-opens=java.base/java.util=${fory_open_targets}" + printf " %s" "--add-opens=java.base/java.util.concurrent=${fory_open_targets}" + printf " %s" "--add-opens=java.base/java.util.concurrent.atomic=${fory_open_targets}" + printf " %s" "--add-opens=java.base/java.io=${fory_open_targets}" + printf " %s" "--add-opens=java.base/java.net=${fory_open_targets}" + printf " %s" "--add-opens=java.base/java.nio=${fory_open_targets}" + printf " %s" "--add-opens=java.base/java.math=${fory_open_targets}" +} + +jdk25_javac_options() { + printf "%s" "--add-opens=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED" + printf " %s" "--add-opens=jdk.compiler/com.sun.tools.javac.processing=ALL-UNNAMED" + printf " %s" "--add-opens=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED" + printf " %s" "--add-opens=jdk.compiler/com.sun.tools.javac.code=ALL-UNNAMED" + printf " %s" "--add-opens=jdk.compiler/com.sun.tools.javac.comp=ALL-UNNAMED" + printf " %s" "--add-opens=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED" + printf " %s" "--add-opens=jdk.compiler/com.sun.tools.javac.jvm=ALL-UNNAMED" + printf " %s" "--add-opens=jdk.compiler/com.sun.tools.javac.main=ALL-UNNAMED" + printf " %s" "--add-opens=jdk.compiler/com.sun.tools.javac.model=ALL-UNNAMED" + printf " %s" "--add-opens=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED" + printf " %s" "--add-opens=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED" +} + +use_jdk() { + local jdk="$1" + export JAVA_HOME="$ROOT/$jdk" + export PATH=$JAVA_HOME/bin:$PATH + if [[ "$jdk" == zulu25* ]]; then + export JDK_JAVA_OPTIONS="$(jdk25_deny_options) $(jdk25_javac_options)" + else + unset JDK_JAVA_OPTIONS + fi +} + +install_jdk25_fory_artifacts() { + local old_java_home="${JAVA_HOME:-}" + local old_path="$PATH" + local old_jdk_java_options="${JDK_JAVA_OPTIONS:-}" + local had_java_home=0 + local had_jdk_java_options=0 + [[ -n "${JAVA_HOME+x}" ]] && had_java_home=1 + [[ -n "${JDK_JAVA_OPTIONS+x}" ]] && had_jdk_java_options=1 + + use_jdk "zulu25.30.17-ca-jdk25.0.1-linux_x64" cd "$ROOT"/java - mvn -T10 -B --no-transfer-progress clean install -DskipTests + mvn -T10 -B --no-transfer-progress clean install -DskipTests -Dmaven.compiler.parameters=true -pl '!:fory-testsuite' + echo "Verify JDK25 benchmark multi-release jar" + cd "$ROOT"/benchmarks/java + mvn -T10 -B --no-transfer-progress -Pjmh -DskipTests install + echo "Verify JPMS tests on JDK25" + cd "$ROOT"/integration_tests/jpms_tests + mvn -T10 -B --no-transfer-progress clean test + + if [[ "$had_java_home" -eq 1 ]]; then + export JAVA_HOME="$old_java_home" + else + unset JAVA_HOME + fi + export PATH="$old_path" + if [[ "$had_jdk_java_options" -eq 1 ]]; then + export JDK_JAVA_OPTIONS="$old_jdk_java_options" + else + unset JDK_JAVA_OPTIONS + fi +} + +integration_tests() { + install_jdk25_fory_artifacts echo "benchmark tests" cd "$ROOT"/benchmarks/java mvn -T10 -B --no-transfer-progress clean test install -Pjmh echo "Start JPMS tests" cd "$ROOT"/integration_tests/jpms_tests - mvn -T10 -B --no-transfer-progress clean compile + mvn -T10 -B --no-transfer-progress clean test echo "Start jdk compatibility tests" cd "$ROOT"/integration_tests/jdk_compatibility_tests mvn -T10 -B --no-transfer-progress clean test for jdk in "${JDKS[@]}"; do - export JAVA_HOME="$ROOT/$jdk" - export PATH=$JAVA_HOME/bin:$PATH + use_jdk "$jdk" echo "First round for generate data: ${jdk}" mvn -T10 --no-transfer-progress clean test -Dtest=org.apache.fory.integration_tests.JDKCompatibilityTest done for jdk in "${JDKS[@]}"; do - export JAVA_HOME="$ROOT/$jdk" - export PATH=$JAVA_HOME/bin:$PATH + use_jdk "$jdk" echo "Second round for compatibility: ${jdk}" mvn -T10 --no-transfer-progress clean test -Dtest=org.apache.fory.integration_tests.JDKCompatibilityTest done @@ -117,11 +199,25 @@ integration_tests() { jdk17_plus_tests() { java -version - export JDK_JAVA_OPTIONS="--add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED" + java_version=$(java -version 2>&1 | awk -F '"' '/version/ {print $2; exit}') + if [[ "$java_version" == 1.* ]]; then + java_major=$(echo "$java_version" | cut -d. -f2) + else + java_major=$(echo "$java_version" | cut -d. -f1) + fi + JDK_JAVA_OPTIONS="--add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED" + if [[ "$java_major" -ge 25 ]]; then + JDK_JAVA_OPTIONS="$JDK_JAVA_OPTIONS $(jdk25_deny_options) $(jdk25_javac_options)" + fi + export JDK_JAVA_OPTIONS echo "Executing fory java tests" cd "$ROOT/java" set +e - mvn -T10 --batch-mode --no-transfer-progress install + jdk25_test_classpath=() + if [[ "$java_major" -ge 25 ]]; then + jdk25_test_classpath=(-Dfory.jdk25.test.classpath=true -Dmaven.compiler.parameters=true) + fi + mvn -T10 --batch-mode --no-transfer-progress install "${jdk25_test_classpath[@]}" testcode=$? if [[ $testcode -ne 0 ]]; then exit $testcode diff --git a/ci/tasks/java.py b/ci/tasks/java.py index c42cd39627..cf8bb10041 100644 --- a/ci/tasks/java.py +++ b/ci/tasks/java.py @@ -76,6 +76,93 @@ def install_jdks(): logging.info("JDKs downloaded and installed successfully") +def jdk25_deny_options(): + fory_open_targets = "ALL-UNNAMED,org.apache.fory.core,org.apache.fory.format" + return [ + "--sun-misc-unsafe-memory-access=deny", + f"--add-opens=java.base/java.lang={fory_open_targets}", + f"--add-opens=java.base/java.lang.invoke={fory_open_targets}", + f"--add-opens=java.base/java.lang.reflect={fory_open_targets}", + f"--add-opens=java.base/jdk.internal.reflect={fory_open_targets}", + f"--add-opens=java.base/java.util={fory_open_targets}", + f"--add-opens=java.base/java.util.concurrent={fory_open_targets}", + f"--add-opens=java.base/java.util.concurrent.atomic={fory_open_targets}", + f"--add-opens=java.base/java.io={fory_open_targets}", + f"--add-opens=java.base/java.net={fory_open_targets}", + f"--add-opens=java.base/java.nio={fory_open_targets}", + f"--add-opens=java.base/java.math={fory_open_targets}", + ] + + +def jdk25_javac_options(): + return [ + "--add-opens=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED", + "--add-opens=jdk.compiler/com.sun.tools.javac.processing=ALL-UNNAMED", + "--add-opens=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED", + "--add-opens=jdk.compiler/com.sun.tools.javac.code=ALL-UNNAMED", + "--add-opens=jdk.compiler/com.sun.tools.javac.comp=ALL-UNNAMED", + "--add-opens=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED", + "--add-opens=jdk.compiler/com.sun.tools.javac.jvm=ALL-UNNAMED", + "--add-opens=jdk.compiler/com.sun.tools.javac.main=ALL-UNNAMED", + "--add-opens=jdk.compiler/com.sun.tools.javac.model=ALL-UNNAMED", + "--add-opens=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED", + "--add-opens=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED", + ] + + +def set_jdk_options(java_version): + if java_version == "25": + os.environ["JDK_JAVA_OPTIONS"] = " ".join( + jdk25_deny_options() + jdk25_javac_options() + ) + else: + os.environ.pop("JDK_JAVA_OPTIONS", None) + + +def use_jdk(java_version): + java_home = os.path.join(common.PROJECT_ROOT_DIR, JDKS[java_version]) + os.environ["JAVA_HOME"] = java_home + os.environ["PATH"] = f"{java_home}/bin:{os.environ.get('PATH', '')}" + set_jdk_options(java_version) + + +def save_java_env(): + return { + "JAVA_HOME": os.environ.get("JAVA_HOME"), + "PATH": os.environ.get("PATH"), + "JDK_JAVA_OPTIONS": os.environ.get("JDK_JAVA_OPTIONS"), + } + + +def restore_java_env(env): + for key, value in env.items(): + if value is None: + os.environ.pop(key, None) + else: + os.environ[key] = value + + +def install_jdk25_fory_artifacts(): + env = save_java_env() + try: + use_jdk("25") + common.cd_project_subdir("java") + common.exec_cmd( + "mvn -T10 -B --no-transfer-progress clean install -DskipTests " + "-Dmaven.compiler.parameters=true -pl '!:fory-testsuite'" + ) + logging.info("Verify JDK25 benchmark multi-release jar") + common.cd_project_subdir("benchmarks/java") + common.exec_cmd( + "mvn -T10 -B --no-transfer-progress -Pjmh -DskipTests install" + ) + logging.info("Verify JPMS tests on JDK25") + common.cd_project_subdir("integration_tests/jpms_tests") + common.exec_cmd("mvn -T10 -B --no-transfer-progress clean test") + finally: + restore_java_env(env) + + def create_toolchains_xml(jdk_mappings): """Create toolchains.xml file in ~/.m2/ directory.""" import os @@ -167,12 +254,23 @@ def run_jdk17_plus(java_version="17"): """Run Java 17+ tests.""" logging.info(f"Executing fory java tests with Java {java_version}") common.exec_cmd("java -version") - os.environ["JDK_JAVA_OPTIONS"] = ( + jdk_options = [ "--add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED" - ) + ] + if java_version == "25": + jdk_options.extend(jdk25_deny_options()) + jdk_options.extend(jdk25_javac_options()) + os.environ["JDK_JAVA_OPTIONS"] = " ".join(jdk_options) common.cd_project_subdir("java") - common.exec_cmd("mvn -T10 --batch-mode --no-transfer-progress install") + jdk25_test_classpath = "" + if java_version == "25": + jdk25_test_classpath = ( + " -Dfory.jdk25.test.classpath=true -Dmaven.compiler.parameters=true" + ) + common.exec_cmd( + f"mvn -T10 --batch-mode --no-transfer-progress install{jdk25_test_classpath}" + ) logging.info("Executing fory java tests succeeds") @@ -203,10 +301,7 @@ def run_integration_tests(): logging.info("Executing fory integration tests") - common.cd_project_subdir("java") - common.exec_cmd( - "mvn -T10 -B --no-transfer-progress clean install -DskipTests -pl '!:fory-testsuite'" - ) + install_jdk25_fory_artifacts() logging.info("benchmark tests") common.cd_project_subdir("benchmarks/java") @@ -214,7 +309,7 @@ def run_integration_tests(): logging.info("Start JPMS tests") common.cd_project_subdir("integration_tests/jpms_tests") - common.exec_cmd("mvn -T10 -B --no-transfer-progress clean compile") + common.exec_cmd("mvn -T10 -B --no-transfer-progress clean test") logging.info("Start jdk compatibility tests") common.cd_project_subdir("integration_tests/jdk_compatibility_tests") @@ -227,10 +322,8 @@ def run_integration_tests(): # First round: Generate serialized data files logging.info("First round: Generate serialized data files for each JDK version") - for jdk in JDKS.values(): - java_home = os.path.join(common.PROJECT_ROOT_DIR, jdk) - os.environ["JAVA_HOME"] = java_home - os.environ["PATH"] = f"{java_home}/bin:{os.environ.get('PATH', '')}" + for java_version, jdk in JDKS.items(): + use_jdk(java_version) logging.info(f"Generating data with JDK: {jdk}") common.exec_cmd( @@ -239,10 +332,8 @@ def run_integration_tests(): # Second round: Test cross-JDK compatibility logging.info("Second round: Test cross-JDK compatibility") - for jdk in JDKS.values(): - java_home = os.path.join(common.PROJECT_ROOT_DIR, jdk) - os.environ["JAVA_HOME"] = java_home - os.environ["PATH"] = f"{java_home}/bin:{os.environ.get('PATH', '')}" + for java_version, jdk in JDKS.items(): + use_jdk(java_version) logging.info(f"Testing compatibility with JDK: {jdk}") common.exec_cmd( @@ -255,6 +346,13 @@ def run_integration_tests(): def run_graalvm_test(): """Run GraalVM tests.""" logging.info("Start GraalVM tests") + java_major = get_jdk_major_version() + if java_major is not None and java_major >= 25: + os.environ["JDK_JAVA_OPTIONS"] = " ".join( + jdk25_deny_options() + jdk25_javac_options() + ) + else: + os.environ.pop("JDK_JAVA_OPTIONS", None) common.cd_project_subdir("java") common.exec_cmd( @@ -275,6 +373,11 @@ def run_graalvm_test(): def run_release(): """Release to Maven Central.""" logging.info("Starting release to Maven Central with Java") + java_major = get_jdk_major_version() + if java_major is None or java_major < 25: + raise RuntimeError( + "Java releases must run on JDK25+ so MR-JAR entries are packaged" + ) common.cd_project_subdir("java") # Clean and install without tests first diff --git a/docs/guide/java/troubleshooting.md b/docs/guide/java/troubleshooting.md index e04a325185..f4837e211c 100644 --- a/docs/guide/java/troubleshooting.md +++ b/docs/guide/java/troubleshooting.md @@ -148,6 +148,50 @@ Fory fory = Fory.builder() fory.registerSerializer(MyClass.class, new MyClassSerializer(fory.getTypeResolver())); ``` +### JDK25+ zero-Unsafe mode and module opens + +When running on JDK25+ with Unsafe memory access denied, start the JVM with: + +```bash +--sun-misc-unsafe-memory-access=deny +``` + +If Fory needs private fields in your named module, open the target package to both Java modules. +When any Fory artifact is on the classpath instead of the module path, also include `ALL-UNNAMED`: + +```bash +--add-opens=/=ALL-UNNAMED,org.apache.fory.core,org.apache.fory.format +``` + +Some optimized serializers and direct-buffer helpers also need JDK-private packages. Add only the +opens needed by the paths used in your process: + +| Path | Required opens | +| -------------------------------------------------------------------------------- | --------------------------------------------------------------- | +| String fast paths and throwable fields | `java.base/java.lang` | +| Serialized lambdas | `java.base/java.lang.invoke` | +| Reflection-based object construction | `java.base/java.lang.reflect`, `java.base/jdk.internal.reflect` | +| Collection wrappers, sublists, `EnumMap`, and `StringTokenizer` | `java.base/java.util` | +| Blocking queue capacity serializers | `java.base/java.util.concurrent` | +| `ByteArrayInputStream`, `ByteArrayOutputStream`, and Java object-stream metadata | `java.base/java.io` | +| Proxy serializers | `java.base/java.lang.reflect` | +| Direct `ByteBuffer` wrapping | `java.base/java.nio` | + +For example, direct `ByteBuffer` wrapping on the module path requires: + +```bash +--add-opens=java.base/java.nio=ALL-UNNAMED,org.apache.fory.core,org.apache.fory.format +``` + +Normal classes with final instance fields need a constructor that covers those final fields when +Unsafe allocation is denied. Annotate the constructor with +`java.beans.ConstructorProperties`, or compile the class with `-parameters` so Fory can bind +constructor parameters to fields. Non-final fields can still be restored after construction. + +The vectorized Arrow APIs in `fory-format` depend on Apache Arrow's memory layer. With the current +Arrow dependency, those APIs are unavailable when `--sun-misc-unsafe-memory-access=deny` is set +because Arrow initializes its own `sun.misc.Unsafe` memory access internally. + ## Performance Issues ### Slow Initial Serialization diff --git a/integration_tests/graalvm_tests/pom.xml b/integration_tests/graalvm_tests/pom.xml index 38d3e5b50f..a0de283a90 100644 --- a/integration_tests/graalvm_tests/pom.xml +++ b/integration_tests/graalvm_tests/pom.xml @@ -176,6 +176,17 @@ -H:+UnlockExperimentalVMOptions + -J--add-opens=java.base/java.lang=ALL-UNNAMED + -J--add-opens=java.base/java.lang.invoke=ALL-UNNAMED + -J--add-opens=java.base/java.lang.reflect=ALL-UNNAMED + -J--add-opens=java.base/jdk.internal.reflect=ALL-UNNAMED + -J--add-opens=java.base/java.util=ALL-UNNAMED + -J--add-opens=java.base/java.util.concurrent=ALL-UNNAMED + -J--add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED + -J--add-opens=java.base/java.io=ALL-UNNAMED + -J--add-opens=java.base/java.net=ALL-UNNAMED + -J--add-opens=java.base/java.nio=ALL-UNNAMED + -J--add-opens=java.base/java.math=ALL-UNNAMED @@ -201,5 +212,27 @@ + + jdk25-native + + [25,) + + + + + org.graalvm.buildtools + native-maven-plugin + + + --sun-misc-unsafe-memory-access=deny + + + -J--sun-misc-unsafe-memory-access=deny + + + + + + diff --git a/integration_tests/graalvm_tests/src/main/java/org/apache/fory/graalvm/FeatureTestExample.java b/integration_tests/graalvm_tests/src/main/java/org/apache/fory/graalvm/FeatureTestExample.java index c39d39412e..d81ac3125d 100644 --- a/integration_tests/graalvm_tests/src/main/java/org/apache/fory/graalvm/FeatureTestExample.java +++ b/integration_tests/graalvm_tests/src/main/java/org/apache/fory/graalvm/FeatureTestExample.java @@ -19,6 +19,7 @@ package org.apache.fory.graalvm; +import java.beans.ConstructorProperties; import java.lang.reflect.InvocationHandler; import java.lang.reflect.Method; import java.lang.reflect.Proxy; @@ -50,6 +51,7 @@ public interface TestInterface { public static class TestInvocationHandler implements InvocationHandler { private final String value; + @ConstructorProperties("value") public TestInvocationHandler(String value) { this.value = value; } diff --git a/integration_tests/graalvm_tests/src/main/java/org/apache/fory/graalvm/Foo.java b/integration_tests/graalvm_tests/src/main/java/org/apache/fory/graalvm/Foo.java index 3aa1362faf..e5d4a0065f 100644 --- a/integration_tests/graalvm_tests/src/main/java/org/apache/fory/graalvm/Foo.java +++ b/integration_tests/graalvm_tests/src/main/java/org/apache/fory/graalvm/Foo.java @@ -19,6 +19,7 @@ package org.apache.fory.graalvm; +import java.beans.ConstructorProperties; import java.io.Serializable; import java.util.List; import java.util.Map; @@ -30,6 +31,7 @@ public class Foo implements Serializable { List f3; Map f4; + @ConstructorProperties({"f1", "f2", "f3", "f4"}) public Foo(int f1, String f2, List f3, Map f4) { this.f1 = f1; this.f2 = f2; diff --git a/integration_tests/graalvm_tests/src/main/java/org/apache/fory/graalvm/ObjectStreamExample.java b/integration_tests/graalvm_tests/src/main/java/org/apache/fory/graalvm/ObjectStreamExample.java index f7a9c29932..68dca1b20b 100644 --- a/integration_tests/graalvm_tests/src/main/java/org/apache/fory/graalvm/ObjectStreamExample.java +++ b/integration_tests/graalvm_tests/src/main/java/org/apache/fory/graalvm/ObjectStreamExample.java @@ -19,6 +19,7 @@ package org.apache.fory.graalvm; +import java.beans.ConstructorProperties; import java.io.Serializable; import java.util.AbstractMap; import java.util.Arrays; @@ -66,7 +67,16 @@ public class ObjectStreamExample extends AbstractMap { FORY.ensureSerializersCompiled(); } - final int[] ints = new int[10]; + final int[] ints; + + public ObjectStreamExample() { + this(new int[10]); + } + + @ConstructorProperties("ints") + public ObjectStreamExample(int[] ints) { + this.ints = ints; + } public static void main(String[] args) { AsyncTreeSetSubclass values = new AsyncTreeSetSubclass(); diff --git a/integration_tests/jdk_compatibility_tests/.gitignore b/integration_tests/jdk_compatibility_tests/.gitignore new file mode 100644 index 0000000000..5cee0c0b39 --- /dev/null +++ b/integration_tests/jdk_compatibility_tests/.gitignore @@ -0,0 +1,4 @@ +/object_schema_consistent* +/object_schema_compatible* +/custom_object_schema_consistent* +/custom_object_schema_compatible* diff --git a/integration_tests/jdk_compatibility_tests/pom.xml b/integration_tests/jdk_compatibility_tests/pom.xml index b0d4c50797..1d0899d92f 100644 --- a/integration_tests/jdk_compatibility_tests/pom.xml +++ b/integration_tests/jdk_compatibility_tests/pom.xml @@ -81,4 +81,37 @@ + + + jdk25-and-higher + + [25,) + + + + + org.apache.maven.plugins + maven-surefire-plugin + + + --sun-misc-unsafe-memory-access=deny + --add-opens=java.base/java.lang=ALL-UNNAMED + --add-opens=java.base/java.lang.invoke=ALL-UNNAMED + --add-opens=java.base/java.lang.reflect=ALL-UNNAMED + --add-opens=java.base/jdk.internal.reflect=ALL-UNNAMED + --add-opens=java.base/java.util=ALL-UNNAMED + --add-opens=java.base/java.util.concurrent=ALL-UNNAMED + --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED + --add-opens=java.base/java.io=ALL-UNNAMED + --add-opens=java.base/java.net=ALL-UNNAMED + --add-opens=java.base/java.nio=ALL-UNNAMED + --add-opens=java.base/java.math=ALL-UNNAMED + + + + + + + + diff --git a/integration_tests/jpms_tests/pom.xml b/integration_tests/jpms_tests/pom.xml index 1d672155f7..e98b2d633c 100644 --- a/integration_tests/jpms_tests/pom.xml +++ b/integration_tests/jpms_tests/pom.xml @@ -76,4 +76,37 @@ + + + jdk25-and-higher + + [25,) + + + + + org.apache.maven.plugins + maven-surefire-plugin + + + --sun-misc-unsafe-memory-access=deny + --add-opens=java.base/java.lang=org.apache.fory.core,org.apache.fory.format + --add-opens=java.base/java.lang.invoke=org.apache.fory.core,org.apache.fory.format + --add-opens=java.base/java.lang.reflect=org.apache.fory.core,org.apache.fory.format + --add-opens=java.base/jdk.internal.reflect=org.apache.fory.core,org.apache.fory.format + --add-opens=java.base/java.util=org.apache.fory.core,org.apache.fory.format + --add-opens=java.base/java.util.concurrent=org.apache.fory.core,org.apache.fory.format + --add-opens=java.base/java.util.concurrent.atomic=org.apache.fory.core,org.apache.fory.format + --add-opens=java.base/java.io=org.apache.fory.core,org.apache.fory.format + --add-opens=java.base/java.net=org.apache.fory.core,org.apache.fory.format + --add-opens=java.base/java.nio=org.apache.fory.core,org.apache.fory.format + --add-opens=java.base/java.math=org.apache.fory.core,org.apache.fory.format + + + + + + + + diff --git a/integration_tests/jpms_tests/src/main/java/module-info.java b/integration_tests/jpms_tests/src/main/java/module-info.java index 2cde522de1..02eb446d24 100644 --- a/integration_tests/jpms_tests/src/main/java/module-info.java +++ b/integration_tests/jpms_tests/src/main/java/module-info.java @@ -26,4 +26,8 @@ // we can't really test any classes from this module because it only contains test-classes requires org.apache.fory.test.suite; + + exports org.apache.fory.integration_tests.model; + exports org.apache.fory.integration_tests.publicserializer; + opens org.apache.fory.integration_tests.model to org.apache.fory.core, org.apache.fory.format; } diff --git a/integration_tests/jpms_tests/src/main/java/org/apache/fory/integration_tests/model/PrivateFieldBean.java b/integration_tests/jpms_tests/src/main/java/org/apache/fory/integration_tests/model/PrivateFieldBean.java new file mode 100644 index 0000000000..8b0c9e326d --- /dev/null +++ b/integration_tests/jpms_tests/src/main/java/org/apache/fory/integration_tests/model/PrivateFieldBean.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.integration_tests.model; + +public final class PrivateFieldBean { + private int value; + + public PrivateFieldBean(int value) { + this.value = value; + } + + public int value() { + return value; + } +} diff --git a/integration_tests/jpms_tests/src/main/java/org/apache/fory/integration_tests/publicserializer/PublicSerializerValue.java b/integration_tests/jpms_tests/src/main/java/org/apache/fory/integration_tests/publicserializer/PublicSerializerValue.java new file mode 100644 index 0000000000..6b2e131f2d --- /dev/null +++ b/integration_tests/jpms_tests/src/main/java/org/apache/fory/integration_tests/publicserializer/PublicSerializerValue.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.integration_tests.publicserializer; + +public final class PublicSerializerValue { + public final int value; + + public PublicSerializerValue(int value) { + this.value = value; + } +} diff --git a/integration_tests/jpms_tests/src/main/java/org/apache/fory/integration_tests/publicserializer/PublicSerializerValueSerializer.java b/integration_tests/jpms_tests/src/main/java/org/apache/fory/integration_tests/publicserializer/PublicSerializerValueSerializer.java new file mode 100644 index 0000000000..25b2aa2e43 --- /dev/null +++ b/integration_tests/jpms_tests/src/main/java/org/apache/fory/integration_tests/publicserializer/PublicSerializerValueSerializer.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.integration_tests.publicserializer; + +import org.apache.fory.context.ReadContext; +import org.apache.fory.context.WriteContext; +import org.apache.fory.serializer.Serializer; +import org.apache.fory.resolver.TypeResolver; + +public final class PublicSerializerValueSerializer extends Serializer { + public PublicSerializerValueSerializer(TypeResolver typeResolver, Class type) { + super(typeResolver.getConfig(), type); + } + + @Override + public void write(WriteContext writeContext, PublicSerializerValue value) { + writeContext.getBuffer().writeInt32(value.value); + } + + @Override + public PublicSerializerValue read(ReadContext readContext) { + return new PublicSerializerValue(readContext.getBuffer().readInt32()); + } +} diff --git a/integration_tests/jpms_tests/src/test/java/org/apache/fory/integration_tests/JpmsFieldAccessorTest.java b/integration_tests/jpms_tests/src/test/java/org/apache/fory/integration_tests/JpmsFieldAccessorTest.java new file mode 100644 index 0000000000..fdfbe0d49e --- /dev/null +++ b/integration_tests/jpms_tests/src/test/java/org/apache/fory/integration_tests/JpmsFieldAccessorTest.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.integration_tests; + +import java.lang.reflect.Field; +import org.apache.fory.Fory; +import org.apache.fory.integration_tests.model.PrivateFieldBean; +import org.apache.fory.integration_tests.publicserializer.PublicSerializerValue; +import org.apache.fory.integration_tests.publicserializer.PublicSerializerValueSerializer; +import org.apache.fory.reflect.FieldAccessor; +import org.testng.Assert; +import org.testng.annotations.Test; + +public class JpmsFieldAccessorTest { + @Test + public void testPrivateFieldAccess() throws Exception { + PrivateFieldBean bean = new PrivateFieldBean(7); + Field field = PrivateFieldBean.class.getDeclaredField("value"); + FieldAccessor accessor = FieldAccessor.createAccessor(field); + Assert.assertEquals(accessor.getInt(bean), 7); + accessor.putInt(bean, 9); + Assert.assertEquals(bean.value(), 9); + } + + @Test + public void testPublicSerializerInExportedPackage() { + Fory fory = Fory.builder().withXlang(false).requireClassRegistration(false).build(); + fory.registerSerializer(PublicSerializerValue.class, PublicSerializerValueSerializer.class); + PublicSerializerValue result = + (PublicSerializerValue) fory.deserialize(fory.serialize(new PublicSerializerValue(11))); + Assert.assertEquals(result.value, 11); + } +} diff --git a/java/fory-core/pom.xml b/java/fory-core/pom.xml index 4176d74184..f5a9a699ea 100644 --- a/java/fory-core/pom.xml +++ b/java/fory-core/pom.xml @@ -77,6 +77,7 @@ + true org.apache.fory.core @@ -92,6 +93,7 @@ shade + false true @@ -118,6 +120,7 @@ + true org.apache.fory.core @@ -135,6 +138,263 @@ + + jdk25-multi-release + + [25,] + + + + + org.apache.maven.plugins + maven-source-plugin + + + attach-sources + package + + jar + + + + + + org.apache.maven.plugins + maven-antrun-plugin + 3.1.0 + + + compile-jdk25-multi-release + process-classes + + run + + + + + + + + + + + + + + + + + prepare-jdk25-test-classes + process-test-classes + + run + + + + + + + + + + + + + + + + + + + verify-jdk25-multi-release-jar + verify + + run + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + patch-jdk25-source-jar + package + + run + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + ${project.build.directory}/jdk25-test-classes + + + + + xlang-parallel diff --git a/java/fory-core/src/main/java/org/apache/fory/Fory.java b/java/fory-core/src/main/java/org/apache/fory/Fory.java index ed770247a3..3ad373043b 100644 --- a/java/fory-core/src/main/java/org/apache/fory/Fory.java +++ b/java/fory-core/src/main/java/org/apache/fory/Fory.java @@ -563,7 +563,7 @@ public T copy(T obj) { private void serializeToStream(OutputStream outputStream, Consumer function) { MemoryBuffer buf = getBuffer(); - if (MemoryUtils.JDK_INTERNAL_FIELD_ACCESS + if (MemoryUtils.JDK_BYTE_ARRAY_STREAM_FIELD_ACCESS && outputStream.getClass() == ByteArrayOutputStream.class) { byte[] oldBytes = buf.getHeapMemory(); // Note: This should not be null. assert oldBytes != null; diff --git a/java/fory-core/src/main/java/org/apache/fory/builder/CodecBuilder.java b/java/fory-core/src/main/java/org/apache/fory/builder/CodecBuilder.java index f786b10c57..5c598b72ce 100644 --- a/java/fory-core/src/main/java/org/apache/fory/builder/CodecBuilder.java +++ b/java/fory-core/src/main/java/org/apache/fory/builder/CodecBuilder.java @@ -310,7 +310,7 @@ private Expression reflectAccessField( private Expression unsafeAccessField( Expression inputObject, Class cls, Descriptor descriptor) { String fieldName = descriptor.getName(); - Expression fieldOffsetExpr = getFieldOffset(cls, descriptor); + Expression fieldOffsetExpr = fieldOffsetExpr(cls, descriptor); boolean fieldNullable = fieldNullable(descriptor); if (descriptor.getTypeRef().isPrimitive()) { // ex: UnsafeOps.getFloat(obj, fieldOffset) @@ -333,7 +333,7 @@ private Expression unsafeAccessField( } } - private Expression getFieldOffset(Class cls, Descriptor descriptor) { + private Expression fieldOffsetExpr(Class cls, Descriptor descriptor) { Field field = descriptor.getField(); String fieldName = descriptor.getName(); // Use Field in case the class has duplicate field name as `fieldName`. @@ -351,9 +351,7 @@ private Expression getFieldOffset(Class cls, Descriptor descriptor) { .inline(); }); } else { - long fieldOffset = ReflectionUtils.getFieldOffset(field); - Preconditions.checkArgument(fieldOffset != -1); - return Literal.ofLong(fieldOffset); + return Literal.ofLong(UnsafeOps.objectFieldOffset(field)); } } @@ -426,7 +424,7 @@ private Expression reflectSetField(Expression bean, Field field, Expression valu private Expression unsafeSetField(Expression bean, Descriptor descriptor, Expression value) { TypeRef fieldType = descriptor.getTypeRef(); // Use Field in case the class has duplicate field name as `fieldName`. - Expression fieldOffsetExpr = getFieldOffset(beanClass, descriptor); + Expression fieldOffsetExpr = fieldOffsetExpr(beanClass, descriptor); if (descriptor.getTypeRef().isPrimitive()) { Preconditions.checkArgument(getRawType(value.type()) == getRawType(fieldType)); String funcName = "put" + StringUtils.capitalize(getRawType(fieldType).toString()); diff --git a/java/fory-core/src/main/java/org/apache/fory/builder/CompatibleCodecBuilder.java b/java/fory-core/src/main/java/org/apache/fory/builder/CompatibleCodecBuilder.java index 3b44448db5..ffeaab8100 100644 --- a/java/fory-core/src/main/java/org/apache/fory/builder/CompatibleCodecBuilder.java +++ b/java/fory-core/src/main/java/org/apache/fory/builder/CompatibleCodecBuilder.java @@ -42,6 +42,8 @@ import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.meta.TypeDef; import org.apache.fory.platform.GraalvmSupport; +import org.apache.fory.reflect.ObjectCreator; +import org.apache.fory.reflect.ObjectCreators; import org.apache.fory.reflect.TypeRef; import org.apache.fory.resolver.TypeResolver; import org.apache.fory.serializer.CodegenSerializer; @@ -130,6 +132,29 @@ public CompatibleCodecBuilder(TypeRef beanType, Fory fory, TypeDef typeDef) { } this.defaultValueLanguage = defaultValueLanguage; this.defaultValueFields = defaultValueFields; + if (!isRecord) { + initConstructorFields( + sortedDescriptors, + true, + defaultFieldNames(defaultValueFields), + defaultDeclaringClasses(defaultValueFields)); + } + } + + private static String[] defaultFieldNames(DefaultValueUtils.DefaultValueField[] fields) { + String[] names = new String[fields.length]; + for (int i = 0; i < fields.length; i++) { + names[i] = fields[i].getFieldName(); + } + return names; + } + + private static Class[] defaultDeclaringClasses(DefaultValueUtils.DefaultValueField[] fields) { + Class[] declaringClasses = new Class[fields.length]; + for (int i = 0; i < fields.length; i++) { + declaringClasses[i] = fields[i].getDeclaringClass(); + } + return declaringClasses; } // Must be static to be shared across the whole process life. @@ -295,10 +320,25 @@ protected Expression newBean() { Expression.ListExpression setDefaultsExpr = new Expression.ListExpression(); setDefaultsExpr.add(bean); + addDefaultValueSetters(setDefaultsExpr, bean); + setDefaultsExpr.add(bean); + return setDefaultsExpr; + } + + @Override + protected void postCreateConstructorObject( + Expression.ListExpression expressions, Expression bean) { + addDefaultValueSetters(expressions, bean); + } + + private void addDefaultValueSetters(Expression.ListExpression expressions, Expression bean) { Map descriptors = Descriptor.getAllDescriptorsMap(beanClass); for (DefaultValueUtils.DefaultValueField defaultField : defaultValueFields) { Object defaultValue = defaultField.getDefaultValue(); Member member = defaultField.getFieldAccessor().getField(); + if (constructorOwnsField(member)) { + continue; + } Descriptor descriptor = descriptors.get(member); TypeRef typeRef = descriptor.getTypeRef(); Expression defaultValueExpr; @@ -322,9 +362,54 @@ protected Expression newBean() { return new Expression.Cast(expr, typeRef); }); } - setDefaultsExpr.add(super.setFieldValue(bean, descriptor, defaultValueExpr)); + expressions.add(super.setFieldValue(bean, descriptor, defaultValueExpr)); } - setDefaultsExpr.add(bean); - return setDefaultsExpr; + } + + private boolean constructorOwnsField(Member member) { + if (constructorFieldIndexes == null) { + return false; + } + ObjectCreator objectCreator = ObjectCreators.getObjectCreator(beanClass); + String[] names = objectCreator.getConstructorFieldNames(); + Class[] declaringClasses = objectCreator.getConstructorFieldDeclaringClasses(); + for (int i = 0; i < names.length; i++) { + Class declaringClass = declaringClasses == null ? null : declaringClasses[i]; + if (names[i].equals(member.getName()) + && (declaringClass == null || declaringClass == member.getDeclaringClass())) { + return true; + } + } + return false; + } + + @Override + protected Expression defaultConstructorValue(int constructorParameterIndex) { + ObjectCreator objectCreator = ObjectCreators.getObjectCreator(beanClass); + String fieldName = objectCreator.getConstructorFieldNames()[constructorParameterIndex]; + Class[] declaringClasses = objectCreator.getConstructorFieldDeclaringClasses(); + Class declaringClass = + declaringClasses == null ? null : declaringClasses[constructorParameterIndex]; + for (DefaultValueUtils.DefaultValueField defaultField : defaultValueFields) { + if (!defaultField.getFieldName().equals(fieldName) + || (declaringClass != null && defaultField.getDeclaringClass() != declaringClass)) { + continue; + } + Object defaultValue = defaultField.getDefaultValue(); + TypeRef typeRef = TypeRef.of(constructorFieldTypes[constructorParameterIndex]); + if (typeRef.unwrap().isPrimitive() || typeRef.equals(STRING_TYPE)) { + return new Literal(defaultValue, typeRef); + } + String funcName = "get" + defaultValueLanguage + "DefaultValue"; + return new Expression.Cast( + new StaticInvoke( + DefaultValueUtils.class, + funcName, + OBJECT_TYPE, + staticBeanClassExpr(), + Literal.ofString(fieldName)), + typeRef); + } + return super.defaultConstructorValue(constructorParameterIndex); } } diff --git a/java/fory-core/src/main/java/org/apache/fory/builder/ObjectCodecBuilder.java b/java/fory-core/src/main/java/org/apache/fory/builder/ObjectCodecBuilder.java index 5705592d79..2220a53013 100644 --- a/java/fory-core/src/main/java/org/apache/fory/builder/ObjectCodecBuilder.java +++ b/java/fory-core/src/main/java/org/apache/fory/builder/ObjectCodecBuilder.java @@ -37,6 +37,7 @@ import java.util.ArrayList; import java.util.Collection; +import java.util.IdentityHashMap; import java.util.List; import java.util.Map; import java.util.Objects; @@ -46,6 +47,7 @@ import org.apache.fory.codegen.Code; import org.apache.fory.codegen.CodegenContext; import org.apache.fory.codegen.Expression; +import org.apache.fory.codegen.Expression.Cast; import org.apache.fory.codegen.Expression.Inlineable; import org.apache.fory.codegen.Expression.Invoke; import org.apache.fory.codegen.Expression.ListExpression; @@ -59,8 +61,10 @@ import org.apache.fory.logging.LoggerFactory; import org.apache.fory.meta.TypeDef; import org.apache.fory.platform.UnsafeOps; +import org.apache.fory.reflect.ObjectCreator; import org.apache.fory.reflect.ObjectCreators; import org.apache.fory.reflect.TypeRef; +import org.apache.fory.serializer.AbstractObjectSerializer; import org.apache.fory.serializer.ObjectSerializer; import org.apache.fory.type.BFloat16; import org.apache.fory.type.Descriptor; @@ -93,6 +97,10 @@ public class ObjectCodecBuilder extends BaseObjectCodecBuilder { private final Literal classVersionHash; protected ObjectCodecOptimizer objectCodecOptimizer; protected Map recordReversedMapping; + protected Map fieldIndexes; + protected int[] constructorFieldIndexes; + protected boolean[] constructorFieldMask; + protected Class[] constructorFieldTypes; public ObjectCodecBuilder(Class beanClass, Fory fory) { super(TypeRef.of(beanClass), fory, Generated.GeneratedObjectSerializer.class); @@ -133,6 +141,8 @@ public ObjectCodecBuilder(Class beanClass, Fory fory) { buildRecordComponentDefaultValues(); } recordReversedMapping = RecordUtils.buildFieldToComponentMapping(beanClass); + } else { + initConstructorFields(grouper.getSortedDescriptors(), true); } } @@ -147,6 +157,122 @@ protected ObjectCodecBuilder(TypeRef beanType, Fory fory, Class superSeria } } + protected final void initConstructorFields( + List sortedDescriptors, boolean allowMissingNonFinal) { + initConstructorFields(sortedDescriptors, allowMissingNonFinal, null); + } + + protected final void initConstructorFields( + List sortedDescriptors, boolean allowMissingNonFinal, String[] defaultFields) { + initConstructorFields(sortedDescriptors, allowMissingNonFinal, defaultFields, null); + } + + protected final void initConstructorFields( + List sortedDescriptors, + boolean allowMissingNonFinal, + String[] defaultFields, + Class[] defaultDeclaringClasses) { + ObjectCreator objectCreator = ObjectCreators.getObjectCreator(beanClass); + if (!objectCreator.hasConstructorFields()) { + return; + } + fieldIndexes = buildFieldIndexes(sortedDescriptors); + constructorFieldTypes = objectCreator.getConstructorFieldTypes(); + constructorFieldIndexes = + buildConstructorFieldIndexes( + sortedDescriptors, + objectCreator, + allowMissingNonFinal, + defaultFields, + defaultDeclaringClasses); + constructorFieldMask = buildConstructorFieldMask(sortedDescriptors.size()); + } + + private static Map buildFieldIndexes(List descriptors) { + Map indexes = new IdentityHashMap<>(); + for (int i = 0; i < descriptors.size(); i++) { + indexes.put(descriptors.get(i), i); + } + return indexes; + } + + private int[] buildConstructorFieldIndexes( + List descriptors, + ObjectCreator objectCreator, + boolean allowMissingNonFinal, + String[] defaultFields, + Class[] defaultDeclaringClasses) { + String[] names = objectCreator.getConstructorFieldNames(); + Class[] declaringClasses = objectCreator.getConstructorFieldDeclaringClasses(); + boolean[] finalFields = objectCreator.getConstructorFieldFinal(); + int[] indexes = new int[names.length]; + for (int i = 0; i < names.length; i++) { + Class declaringClass = declaringClasses == null ? null : declaringClasses[i]; + boolean allowMissing = + (allowMissingNonFinal && !finalFields[i]) + || contains(defaultFields, defaultDeclaringClasses, names[i], declaringClass); + indexes[i] = constructorFieldIndex(descriptors, declaringClass, names[i], allowMissing); + } + return indexes; + } + + private static boolean contains( + String[] values, Class[] declaringClasses, String value, Class declaringClass) { + if (values == null) { + return false; + } + for (int i = 0; i < values.length; i++) { + if (values[i].equals(value) + && (declaringClasses == null + || i >= declaringClasses.length + || declaringClasses[i] == null + || declaringClasses[i] == declaringClass)) { + return true; + } + } + return false; + } + + private int constructorFieldIndex( + List descriptors, + Class declaringClass, + String fieldName, + boolean allowMissing) { + int index = -1; + for (int i = 0; i < descriptors.size(); i++) { + Descriptor descriptor = descriptors.get(i); + if (!descriptor.getName().equals(fieldName) + || (declaringClass != null + && (descriptor.getField() == null + || descriptor.getField().getDeclaringClass() != declaringClass))) { + continue; + } + if (index >= 0) { + throw new IllegalStateException( + "Constructor field " + fieldName + " is ambiguous for " + beanClass); + } + index = i; + } + if (index < 0) { + if (allowMissing) { + return -1; + } + throw new IllegalStateException( + "Constructor field " + fieldName + " is not serialized for " + beanClass); + } + return index; + } + + private boolean[] buildConstructorFieldMask(int size) { + boolean[] mask = new boolean[size]; + for (int index : constructorFieldIndexes) { + if (index >= 0) { + mask[index] = true; + } + } + return mask; + } + @Override protected String codecSuffix() { return ""; @@ -548,12 +674,20 @@ public Expression buildDecodeExpression() { if (typeResolver.checkClassVersion()) { expressions.add(checkClassVersion(buffer)); } + if (!isRecord && constructorFieldIndexes != null) { + return buildConstructorDecodeExpression(buffer, expressions); + } Expression bean; if (!isRecord) { - bean = newBean(); - Expression referenceObject = invokeReadContext("reference", bean); - expressions.add(bean); - expressions.add(referenceObject); + if (constructorFieldIndexes == null) { + bean = newBean(); + Expression referenceObject = invokeReadContext("reference", bean); + expressions.add(bean); + expressions.add(referenceObject); + } else { + bean = new FieldsArray(fieldIndexes.size()); + expressions.add(bean); + } } else { if (recordCtrAccessible) { bean = new FieldsCollector(); @@ -582,6 +716,132 @@ public Expression buildDecodeExpression() { return expressions; } + private Expression buildConstructorDecodeExpression( + Reference buffer, ListExpression expressions) { + FieldsArray fieldsArray = new FieldsArray(fieldIndexes.size()); + expressions.add(fieldsArray); + expressions.add( + new StaticInvoke( + AbstractObjectSerializer.class, + "beginConstructorRef", + PRIMITIVE_VOID_TYPE, + readContextRef())); + List bufferedNonConstructorFields = new ArrayList<>(); + int remainingConstructorFields = countConstructorFields(); + Expression bean = null; + if (remainingConstructorFields == 0) { + bean = createCtorBean(expressions, fieldsArray); + } + for (Descriptor descriptor : protocolDescriptors()) { + int index = fieldIndexes.get(descriptor); + walkPath.add(descriptor.getDeclaringClass() + descriptor.getName()); + if (constructorFieldMask[index]) { + expressions.add(deserializeToFieldsArray(fieldsArray, buffer, descriptor, true)); + remainingConstructorFields--; + if (remainingConstructorFields == 0) { + bean = createCtorBean(expressions, fieldsArray); + addBufferedFieldSetters(expressions, bean, fieldsArray, bufferedNonConstructorFields); + } + } else if (bean == null) { + expressions.add(deserializeToFieldsArray(fieldsArray, buffer, descriptor, false)); + bufferedNonConstructorFields.add(descriptor); + } else { + expressions.add(deserializeToBean(bean, buffer, descriptor)); + } + walkPath.removeLast(); + } + expressions.add( + new StaticInvoke( + AbstractObjectSerializer.class, + "endConstructorRef", + PRIMITIVE_VOID_TYPE, + readContextRef())); + expressions.add(new Expression.Return(bean)); + return expressions; + } + + private int countConstructorFields() { + int count = 0; + for (boolean constructorField : constructorFieldMask) { + if (constructorField) { + count++; + } + } + return count; + } + + private List protocolDescriptors() { + List descriptors = new ArrayList<>(); + addDescriptors(descriptors, objectCodecOptimizer.primitiveGroups); + addDescriptors(descriptors, objectCodecOptimizer.boxedReadGroups); + addDescriptors(descriptors, objectCodecOptimizer.nonPrimitiveReadGroups); + return descriptors; + } + + private void addDescriptors(List descriptors, List> groups) { + for (List group : groups) { + descriptors.addAll(group); + } + } + + private Expression createCtorBean(ListExpression expressions, FieldsArray fieldsArray) { + Expression bean = createConstructorObject(fieldsArray); + expressions.add( + new StaticInvoke( + AbstractObjectSerializer.class, + "checkNoUnresolvedReadRef", + PRIMITIVE_VOID_TYPE, + readContextRef(), + staticBeanClassExpr())); + expressions.add(bean); + expressions.add( + new StaticInvoke( + AbstractObjectSerializer.class, + "referenceConstructorRef", + PRIMITIVE_VOID_TYPE, + readContextRef(), + bean)); + postCreateConstructorObject(expressions, bean); + return bean; + } + + private Expression deserializeToFieldsArray( + FieldsArray fieldsArray, Reference buffer, Descriptor descriptor, boolean constructorField) { + TypeRef castTypeRef = + hasCompatibleCollectionArrayRead(descriptor) + ? compatibleReadTargetTypeRef(descriptor) + : descriptor.getTypeRef(); + return deserializeField( + buffer, + descriptor, + expr -> { + Expression value = + constructorField ? tryInlineCast(expr, castTypeRef) : new Cast(expr, OBJECT_TYPE); + value = + new StaticInvoke( + AbstractObjectSerializer.class, + constructorField ? "ctorFieldValue" : "bufferFieldValue", + OBJECT_TYPE, + readContextRef(), + value, + staticBeanClassExpr()); + return setFieldValue(fieldsArray, descriptor, value); + }); + } + + private Expression deserializeToBean(Expression bean, Reference buffer, Descriptor descriptor) { + TypeRef castTypeRef = + hasCompatibleCollectionArrayRead(descriptor) + ? compatibleReadTargetTypeRef(descriptor) + : descriptor.getTypeRef(); + return deserializeField( + buffer, + descriptor, + expr -> setFieldValue(bean, descriptor, tryInlineCast(expr, castTypeRef))); + } + + protected void postCreateConstructorObject(ListExpression expressions, Expression bean) {} + protected void deserializeReadGroup( List> readGroups, int numGroups, @@ -607,6 +867,88 @@ protected Expression createRecord(SortedMap recordComponent return new NewInstance(beanType, params); } + protected Expression createConstructorObject(FieldsArray fieldValues) { + Expression[] params = new Expression[constructorFieldIndexes.length]; + for (int i = 0; i < constructorFieldIndexes.length; i++) { + int index = constructorFieldIndexes[i]; + if (index < 0) { + params[i] = defaultConstructorValue(i); + } else { + params[i] = fieldValue(fieldValues, index); + } + } + Expression args = new Expression.NewArray(OBJECT_ARRAY_TYPE, params); + ObjectCreators.getObjectCreator(beanClass); // trigger cache and make error raised early + Expression newInstance = + new Invoke(getObjectCreator(beanClass), "newInstanceWithArguments", OBJECT_TYPE, args); + return sourcePublicAccessible(beanClass) ? new Cast(newInstance, beanType) : newInstance; + } + + protected Expression defaultConstructorValue(int constructorParameterIndex) { + return new StaticInvoke( + AbstractObjectSerializer.class, + "defaultConstructorValue", + OBJECT_TYPE, + staticClassFieldExpr( + constructorFieldTypes[constructorParameterIndex], + "constructorFieldClass" + constructorParameterIndex + "_")); + } + + private void addNonConstructorFieldSetters( + ListExpression expressions, Expression bean, FieldsArray fieldValues) { + for (Descriptor descriptor : objectCodecOptimizer.descriptorGrouper.getSortedDescriptors()) { + int index = fieldIndexes.get(descriptor); + if (constructorFieldMask[index]) { + continue; + } + TypeRef castTypeRef = + hasCompatibleCollectionArrayRead(descriptor) + ? compatibleReadTargetTypeRef(descriptor) + : descriptor.getTypeRef(); + Expression value = + new StaticInvoke( + AbstractObjectSerializer.class, + "resolveBufferedValue", + OBJECT_TYPE, + fieldValue(fieldValues, index), + bean); + value = tryInlineCast(value, castTypeRef); + expressions.add(setFieldValue(bean, descriptor, value)); + } + } + + private void addBufferedFieldSetters( + ListExpression expressions, + Expression bean, + FieldsArray fieldValues, + List descriptors) { + for (Descriptor descriptor : descriptors) { + int index = fieldIndexes.get(descriptor); + TypeRef castTypeRef = + hasCompatibleCollectionArrayRead(descriptor) + ? compatibleReadTargetTypeRef(descriptor) + : descriptor.getTypeRef(); + Expression value = + new StaticInvoke( + AbstractObjectSerializer.class, + "resolveBufferedValue", + OBJECT_TYPE, + fieldValue(fieldValues, index), + bean); + value = tryInlineCast(value, castTypeRef); + expressions.add(setFieldValue(bean, descriptor, value)); + } + } + + private Expression fieldValue(Expression fieldValues, int index) { + return new StaticInvoke( + AbstractObjectSerializer.class, + "fieldValue", + OBJECT_TYPE, + fieldValues, + Literal.ofInt(index)); + } + private class FieldsCollector extends Expression.AbstractExpression { private final TreeMap recordValuesMap = new TreeMap<>(); @@ -625,8 +967,38 @@ public Code.ExprCode doGenCode(CodegenContext ctx) { } } + protected class FieldsArray extends Expression.AbstractExpression { + private final int size; + private final String name; + + protected FieldsArray(int size) { + super(new Expression[0]); + this.size = size; + name = ctx.newName("fieldValues"); + } + + @Override + public TypeRef type() { + return OBJECT_ARRAY_TYPE; + } + + @Override + public Code.ExprCode doGenCode(CodegenContext ctx) { + String code = ctx.type(Object[].class) + " " + name + " = new Object[" + size + "];"; + return new Code.ExprCode(code, FalseLiteral, Code.variable(Object[].class, name)); + } + + int fieldIndex(Descriptor descriptor) { + return fieldIndexes.get(descriptor); + } + } + @Override protected Expression setFieldValue(Expression bean, Descriptor d, Expression value) { + if (bean instanceof FieldsArray) { + return new Expression.AssignArrayElem( + bean, value, Literal.ofInt(((FieldsArray) bean).fieldIndex(d))); + } if (isRecord) { if (recordCtrAccessible) { if (value instanceof Inlineable) { diff --git a/java/fory-core/src/main/java/org/apache/fory/collection/MapEntry.java b/java/fory-core/src/main/java/org/apache/fory/collection/MapEntry.java index b0bc8c3337..5ec07a7974 100644 --- a/java/fory-core/src/main/java/org/apache/fory/collection/MapEntry.java +++ b/java/fory-core/src/main/java/org/apache/fory/collection/MapEntry.java @@ -19,6 +19,7 @@ package org.apache.fory.collection; +import java.beans.ConstructorProperties; import java.util.Map; import java.util.Objects; @@ -27,6 +28,7 @@ public class MapEntry implements Map.Entry { private final K key; private V value; + @ConstructorProperties({"key", "value"}) public MapEntry(K key, V value) { this.key = key; this.value = value; diff --git a/java/fory-core/src/main/java/org/apache/fory/context/CopyContext.java b/java/fory-core/src/main/java/org/apache/fory/context/CopyContext.java index 5eea194405..f17c170ca7 100644 --- a/java/fory-core/src/main/java/org/apache/fory/context/CopyContext.java +++ b/java/fory-core/src/main/java/org/apache/fory/context/CopyContext.java @@ -21,6 +21,7 @@ import java.util.Arrays; import org.apache.fory.collection.IdentityMap; +import org.apache.fory.exception.ForyException; import org.apache.fory.resolver.ClassResolver; import org.apache.fory.resolver.TypeInfo; import org.apache.fory.resolver.TypeResolver; @@ -41,6 +42,7 @@ public final class CopyContext { private final TypeResolver typeResolver; private final boolean copyRefTracking; private final IdentityMap originToCopyMap; + private static final Object COPY_IN_PROGRESS = new Object(); /** * Creates a copy context for one runtime. @@ -88,7 +90,26 @@ public void reference(T origin, T copied) { /** Returns the previously registered copy for {@code origin}, or {@code null} if absent. */ public T getCopyObject(T origin) { - return (T) originToCopyMap.get(origin); + Object copied = originToCopyMap.get(origin); + if (copied == COPY_IN_PROGRESS) { + throw new ForyException( + "Cyclic references to constructor-bound objects cannot be copied before construction."); + } + return (T) copied; + } + + /** Marks {@code origin} as being copied before a constructor-bound copy can be registered. */ + public void markCopying(Object origin) { + if (copyRefTracking && origin != null) { + originToCopyMap.put(origin, COPY_IN_PROGRESS); + } + } + + /** Clears an in-progress constructor-bound copy marker after a failed copy. */ + public void cancelCopy(Object origin) { + if (copyRefTracking && origin != null && originToCopyMap.get(origin) == COPY_IN_PROGRESS) { + originToCopyMap.remove(origin); + } } /** diff --git a/java/fory-core/src/main/java/org/apache/fory/context/MapRefReader.java b/java/fory-core/src/main/java/org/apache/fory/context/MapRefReader.java index 59101b61ac..6613349580 100644 --- a/java/fory-core/src/main/java/org/apache/fory/context/MapRefReader.java +++ b/java/fory-core/src/main/java/org/apache/fory/context/MapRefReader.java @@ -38,6 +38,8 @@ public final class MapRefReader implements RefReader { private long readTotalObjectSize = 0; private final ObjectArray readObjects = new ObjectArray(DEFAULT_ARRAY_CAPACITY); private final IntArray readRefIds = new IntArray(DEFAULT_ARRAY_CAPACITY); + private final IntArray trackedRefIds = new IntArray(DEFAULT_ARRAY_CAPACITY); + private final IntArray unresolvedRefIds = new IntArray(DEFAULT_ARRAY_CAPACITY); private Object readObject; /** Reads a ref-or-null header and resolves cached references immediately when present. */ @@ -45,7 +47,7 @@ public final class MapRefReader implements RefReader { public byte readRefOrNull(MemoryBuffer buffer) { byte headFlag = buffer.readByte(); if (headFlag == Fory.REF_FLAG) { - readObject = getReadRef(buffer.readVarUInt32Small14()); + readObject = readRef(buffer.readVarUInt32Small14()); } else { readObject = null; } @@ -73,7 +75,7 @@ public int preserveRefId(int refId) { public int tryPreserveRefId(MemoryBuffer buffer) { byte headFlag = buffer.readByte(); if (headFlag == Fory.REF_FLAG) { - readObject = getReadRef(buffer.readVarUInt32Small14()); + readObject = readRef(buffer.readVarUInt32Small14()); } else { readObject = null; if (headFlag == Fory.REF_VALUE_FLAG) { @@ -104,12 +106,52 @@ public void reference(Object object) { setReadRef(refId, object); } + /** Binds a reserved ref id that may no longer be the top of the pending-id stack. */ + @Override + public void reference(int refId, Object object) { + removePreservedRefId(refId); + setReadRef(refId, object); + } + + private void removePreservedRefId(int refId) { + for (int i = readRefIds.size - 1; i >= 0; i--) { + if (readRefIds.elementData[i] == refId) { + int numMoved = readRefIds.size - i - 1; + if (numMoved > 0) { + System.arraycopy(readRefIds.elementData, i + 1, readRefIds.elementData, i, numMoved); + } + readRefIds.size--; + return; + } + } + } + /** Returns the previously materialized object stored at {@code id}. */ @Override public Object getReadRef(int id) { return readObjects.get(id); } + private Object readRef(int id) { + if (trackedRefIds.size == 0) { + return readObjects.get(id); + } + Object object = readObjects.get(id); + if (object == null && isTrackedRef(id)) { + unresolvedRefIds.add(id); + } + return object; + } + + private boolean isTrackedRef(int id) { + for (int i = trackedRefIds.size - 1; i >= 0; i--) { + if (trackedRefIds.get(i) == id) { + return true; + } + } + return false; + } + /** Returns the object resolved by the last ref header that pointed to an existing instance. */ @Override public Object getReadRef() { @@ -124,6 +166,44 @@ public void setReadRef(int id, Object object) { } } + @Override + public void trackUnresolvedRef(int id) { + trackedRefIds.add(id); + } + + @Override + public boolean hasTrackedRef() { + return trackedRefIds.size > 0; + } + + @Override + public int currentTrackedRefId() { + return trackedRefIds.get(trackedRefIds.size - 1); + } + + @Override + public void untrackUnresolvedRef() { + if (trackedRefIds.size > 0) { + trackedRefIds.pop(); + } + } + + @Override + public boolean consumeUnresolvedRef(int id) { + boolean found = false; + int newSize = 0; + for (int i = 0; i < unresolvedRefIds.size; i++) { + int unresolvedRefId = unresolvedRefIds.get(i); + if (unresolvedRefId == id) { + found = true; + } else { + unresolvedRefIds.elementData[newSize++] = unresolvedRefId; + } + } + unresolvedRefIds.size = newSize; + return found; + } + /** Exposes the resolved read-reference table for debugging and focused tests. */ public ObjectArray getReadRefs() { return readObjects; @@ -146,6 +226,8 @@ public void reset() { } readObjects.clearApproximate(avg); readRefIds.clear(); + trackedRefIds.clear(); + unresolvedRefIds.clear(); readObject = null; } } diff --git a/java/fory-core/src/main/java/org/apache/fory/context/ReadContext.java b/java/fory-core/src/main/java/org/apache/fory/context/ReadContext.java index 0b03800dd6..6771912454 100644 --- a/java/fory-core/src/main/java/org/apache/fory/context/ReadContext.java +++ b/java/fory-core/src/main/java/org/apache/fory/context/ReadContext.java @@ -360,6 +360,11 @@ public void reference(Object object) { refReader.reference(object); } + /** Binds a specific preserved read ref id to {@code object}. */ + public void reference(int refId, Object object) { + refReader.reference(refId, object); + } + /** Returns a previously read object by ref id. */ public Object getReadRef(int id) { return refReader.getReadRef(id); @@ -375,6 +380,31 @@ public void setReadRef(int id, Object object) { refReader.setReadRef(id, object); } + /** Starts tracking unresolved reads of {@code id} while a constructor-bound object is read. */ + public void trackUnresolvedRef(int id) { + refReader.trackUnresolvedRef(id); + } + + /** Returns whether a constructor-bound object ref id is currently tracked. */ + public boolean hasTrackedRef() { + return refReader.hasTrackedRef(); + } + + /** Returns the active constructor-bound object ref id. */ + public int currentTrackedRefId() { + return refReader.currentTrackedRefId(); + } + + /** Stops tracking unresolved reads for the most recent constructor-bound object. */ + public void untrackUnresolvedRef() { + refReader.untrackUnresolvedRef(); + } + + /** Returns and clears whether {@code id} was read before it was bound to an object. */ + public boolean consumeUnresolvedRef(int id) { + return refReader.consumeUnresolvedRef(id); + } + /** Returns the read-side meta-string state for the current runtime. */ public MetaStringReader getMetaStringReader() { return metaStringReader; diff --git a/java/fory-core/src/main/java/org/apache/fory/context/RefReader.java b/java/fory-core/src/main/java/org/apache/fory/context/RefReader.java index c295bedc38..f18b52973b 100644 --- a/java/fory-core/src/main/java/org/apache/fory/context/RefReader.java +++ b/java/fory-core/src/main/java/org/apache/fory/context/RefReader.java @@ -49,6 +49,11 @@ public interface RefReader { /** Binds the most recently preserved reference id to {@code object}. */ void reference(Object object); + /** Binds a specific preserved reference id to {@code object}. */ + default void reference(int refId, Object object) { + reference(object); + } + /** Returns the previously materialized object for a specific ref id. */ Object getReadRef(int id); @@ -58,6 +63,27 @@ public interface RefReader { /** Replaces the object stored for a previously preserved ref id. */ void setReadRef(int id, Object object); + /** Starts tracking unresolved reads of the currently constructed object's ref id. */ + default void trackUnresolvedRef(int id) {} + + /** Returns whether a constructor-bound object ref id is currently tracked. */ + default boolean hasTrackedRef() { + return false; + } + + /** Returns the most recently tracked constructor-bound object ref id. */ + default int currentTrackedRefId() { + return -1; + } + + /** Stops tracking unresolved reads for the most recent constructor-bound object. */ + default void untrackUnresolvedRef() {} + + /** Returns and clears whether {@code id} was read before it was bound to an object. */ + default boolean consumeUnresolvedRef(int id) { + return false; + } + /** Clears all per-operation ref-tracking state. */ void reset(); @@ -96,6 +122,9 @@ public boolean hasPreservedRefId() { @Override public void reference(Object object) {} + @Override + public void reference(int refId, Object object) {} + @Override public Object getReadRef(int id) { return null; diff --git a/java/fory-core/src/main/java/org/apache/fory/memory/MemoryUtils.java b/java/fory-core/src/main/java/org/apache/fory/memory/MemoryUtils.java index cf626123b1..81d35bb4d1 100644 --- a/java/fory-core/src/main/java/org/apache/fory/memory/MemoryUtils.java +++ b/java/fory-core/src/main/java/org/apache/fory/memory/MemoryUtils.java @@ -28,10 +28,22 @@ /** Memory utils for fory. */ public class MemoryUtils { // JDK25+ internal-field access must be backed by supported access in the multi-release classes. - // When a JDK25+ path needs java.nio private fields, the JVM must be launched with: - // --add-opens=java.base/java.nio=org.apache.fory.core + // When a JDK25+ path needs JDK private fields, open the needed java.base package to both + // org.apache.fory.core and org.apache.fory.format. public static final boolean JDK_INTERNAL_FIELD_ACCESS = !AndroidSupport.IS_ANDROID && _JDKAccess.JDK_INTERNAL_FIELD_ACCESS; + public static final boolean JDK_LANG_FIELD_ACCESS = + !AndroidSupport.IS_ANDROID && _JDKAccess.JDK_LANG_FIELD_ACCESS; + public static final boolean JDK_BYTE_ARRAY_STREAM_FIELD_ACCESS = + !AndroidSupport.IS_ANDROID && _JDKAccess.JDK_BYTE_ARRAY_STREAM_FIELD_ACCESS; + public static final boolean JDK_OBJECT_STREAM_FIELD_ACCESS = + !AndroidSupport.IS_ANDROID && _JDKAccess.JDK_OBJECT_STREAM_FIELD_ACCESS; + public static final boolean JDK_COLLECTION_FIELD_ACCESS = + !AndroidSupport.IS_ANDROID && _JDKAccess.JDK_COLLECTION_FIELD_ACCESS; + public static final boolean JDK_CONCURRENT_FIELD_ACCESS = + !AndroidSupport.IS_ANDROID && _JDKAccess.JDK_CONCURRENT_FIELD_ACCESS; + public static final boolean JDK_PROXY_FIELD_ACCESS = + !AndroidSupport.IS_ANDROID && _JDKAccess.JDK_PROXY_FIELD_ACCESS; public static MemoryBuffer buffer(int size) { return wrap(new byte[size]); @@ -103,9 +115,11 @@ public static void wrap(ByteArrayInputStream stream, MemoryBuffer buffer) { } private static void checkByteArrayStreamWrap(String streamType) { - if (!JDK_INTERNAL_FIELD_ACCESS) { + if (!JDK_BYTE_ARRAY_STREAM_FIELD_ACCESS) { throw new UnsupportedOperationException( - streamType + " direct wrapping is not supported on this platform"); + streamType + + " direct wrapping requires JDK internal field access. On JDK25+, open " + + "java.base/java.io to org.apache.fory.core,org.apache.fory.format."); } } diff --git a/java/fory-core/src/main/java/org/apache/fory/meta/TypeDef.java b/java/fory-core/src/main/java/org/apache/fory/meta/TypeDef.java index 86a82e3e37..26865cb32e 100644 --- a/java/fory-core/src/main/java/org/apache/fory/meta/TypeDef.java +++ b/java/fory-core/src/main/java/org/apache/fory/meta/TypeDef.java @@ -19,7 +19,6 @@ package org.apache.fory.meta; - import java.io.ObjectStreamClass; import java.io.Serializable; import java.lang.reflect.Field; @@ -36,7 +35,6 @@ import org.apache.fory.logging.Logger; import org.apache.fory.logging.LoggerFactory; import org.apache.fory.memory.MemoryBuffer; -import org.apache.fory.reflect.ReflectionUtils; import org.apache.fory.resolver.ClassResolver; import org.apache.fory.resolver.SharedRegistry; import org.apache.fory.resolver.TypeResolver; @@ -64,7 +62,6 @@ * @see ForyBuilder#withCompatible(boolean) * @see CompatibleSerializer * @see ForyBuilder#withMetaShare - * @see ReflectionUtils#getFieldOffset */ public class TypeDef implements Serializable { private static final Logger LOG = LoggerFactory.getLogger(TypeDef.class); diff --git a/java/fory-core/src/main/java/org/apache/fory/platform/UnsafeOps.java b/java/fory-core/src/main/java/org/apache/fory/platform/UnsafeOps.java index 8e214946bc..2dc5456a63 100644 --- a/java/fory-core/src/main/java/org/apache/fory/platform/UnsafeOps.java +++ b/java/fory-core/src/main/java/org/apache/fory/platform/UnsafeOps.java @@ -197,6 +197,12 @@ public static void copyMemory( } } + public static Object[] copyObjectArray(Object[] arr) { + Object[] objects = new Object[arr.length]; + System.arraycopy(arr, 0, objects, 0, arr.length); + return objects; + } + /** Create an instance of type. This method don't call constructor. */ public static T newInstance(Class type) { try { diff --git a/java/fory-core/src/main/java/org/apache/fory/platform/internal/DefineClass.java b/java/fory-core/src/main/java/org/apache/fory/platform/internal/DefineClass.java index c47ffa9d44..c340e673a7 100644 --- a/java/fory-core/src/main/java/org/apache/fory/platform/internal/DefineClass.java +++ b/java/fory-core/src/main/java/org/apache/fory/platform/internal/DefineClass.java @@ -78,4 +78,9 @@ public static Class defineClass( throw new RuntimeException(e); } } + + public static Class defineHiddenNestmate(Class neighbor, byte[] bytecodes) { + throw new UnsupportedOperationException( + "Hidden nestmate class definition requires the JDK25 multi-release DefineClass"); + } } diff --git a/java/fory-core/src/main/java/org/apache/fory/platform/internal/_JDKAccess.java b/java/fory-core/src/main/java/org/apache/fory/platform/internal/_JDKAccess.java index 514a8a9d57..e4c9d47ad9 100644 --- a/java/fory-core/src/main/java/org/apache/fory/platform/internal/_JDKAccess.java +++ b/java/fory-core/src/main/java/org/apache/fory/platform/internal/_JDKAccess.java @@ -21,6 +21,7 @@ import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; +import java.io.ObjectStreamClass; import java.lang.invoke.CallSite; import java.lang.invoke.LambdaMetafactory; import java.lang.invoke.MethodHandle; @@ -30,9 +31,11 @@ import java.lang.reflect.Field; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; +import java.nio.charset.StandardCharsets; import java.util.HashMap; import java.util.Map; import java.util.function.BiConsumer; +import java.util.function.BiFunction; import java.util.function.Consumer; import java.util.function.Function; import java.util.function.Predicate; @@ -63,6 +66,13 @@ public class _JDKAccess { // this API surface and implement supported cases with VarHandle, or set this false so callers // choose public fallbacks. public static final boolean JDK_INTERNAL_FIELD_ACCESS; + public static final boolean JDK_LANG_FIELD_ACCESS; + public static final boolean JDK_STRING_FIELD_ACCESS; + public static final boolean JDK_BYTE_ARRAY_STREAM_FIELD_ACCESS; + public static final boolean JDK_OBJECT_STREAM_FIELD_ACCESS; + public static final boolean JDK_COLLECTION_FIELD_ACCESS; + public static final boolean JDK_CONCURRENT_FIELD_ACCESS; + public static final boolean JDK_PROXY_FIELD_ACCESS; public static final Class _INNER_UNSAFE_CLASS; public static final Object _INNER_UNSAFE; @@ -79,6 +89,13 @@ public class _JDKAccess { } UNSAFE = unsafe; JDK_INTERNAL_FIELD_ACCESS = true; + JDK_LANG_FIELD_ACCESS = true; + JDK_STRING_FIELD_ACCESS = true; + JDK_BYTE_ARRAY_STREAM_FIELD_ACCESS = true; + JDK_OBJECT_STREAM_FIELD_ACCESS = true; + JDK_COLLECTION_FIELD_ACCESS = true; + JDK_CONCURRENT_FIELD_ACCESS = true; + JDK_PROXY_FIELD_ACCESS = true; if (JdkVersion.MAJOR_VERSION >= 11) { try { Field theInternalUnsafeField = Unsafe.class.getDeclaredField("theInternalUnsafe"); @@ -99,9 +116,9 @@ public class _JDKAccess { public static final boolean STRING_VALUE_FIELD_IS_CHARS; public static final boolean STRING_VALUE_FIELD_IS_BYTES; public static final boolean STRING_HAS_COUNT_OFFSET; - private static final long STRING_VALUE_FIELD_OFFSET; - private static final long STRING_COUNT_FIELD_OFFSET; - private static final long STRING_OFFSET_FIELD_OFFSET; + public static final long STRING_VALUE_FIELD_OFFSET; + public static final long STRING_COUNT_FIELD_OFFSET; + public static final long STRING_OFFSET_FIELD_OFFSET; static { try { @@ -150,6 +167,8 @@ private static class StringCoderField { } } + public static final long STRING_CODER_FIELD_OFFSET = StringCoderField.OFFSET; + public static Object getStringValue(String value) { return UNSAFE.getObject(value, STRING_VALUE_FIELD_OFFSET); } @@ -178,6 +197,141 @@ public static Lookup _trustedLookup(Class objectClass) { return lookupCache.get(objectClass, () -> _Lookup._trustedLookup(objectClass)); } + private static final byte LATIN1 = 0; + private static final Byte LATIN1_BOXED = LATIN1; + private static final byte UTF16 = 1; + private static final Byte UTF16_BOXED = UTF16; + private static final MethodHandles.Lookup STRING_LOOK_UP = + JDK_INTERNAL_FIELD_ACCESS ? _trustedLookup(String.class) : null; + private static final BiFunction CHARS_STRING_ZERO_COPY_CTR = + JDK_INTERNAL_FIELD_ACCESS ? getCharsStringZeroCopyCtr() : null; + private static final BiFunction BYTES_STRING_ZERO_COPY_CTR = + JDK_INTERNAL_FIELD_ACCESS ? getBytesStringZeroCopyCtr() : null; + private static final Function LATIN_BYTES_STRING_ZERO_COPY_CTR = + JDK_INTERNAL_FIELD_ACCESS ? getLatinBytesStringZeroCopyCtr() : null; + + public static String newCharsStringZeroCopy(char[] data) { + if (!JDK_INTERNAL_FIELD_ACCESS) { + return new String(data); + } + if (!STRING_VALUE_FIELD_IS_CHARS) { + throw new IllegalStateException("String value isn't char[], current java isn't supported"); + } + return CHARS_STRING_ZERO_COPY_CTR.apply(data, Boolean.TRUE); + } + + public static String newBytesStringZeroCopy(byte coder, byte[] data) { + if (!JDK_INTERNAL_FIELD_ACCESS) { + return newBytesStringSlow(coder, data); + } + if (coder == LATIN1) { + if (LATIN_BYTES_STRING_ZERO_COPY_CTR != null) { + return LATIN_BYTES_STRING_ZERO_COPY_CTR.apply(data); + } + return BYTES_STRING_ZERO_COPY_CTR.apply(data, LATIN1_BOXED); + } else if (coder == UTF16) { + return BYTES_STRING_ZERO_COPY_CTR.apply(data, UTF16_BOXED); + } else { + return BYTES_STRING_ZERO_COPY_CTR.apply(data, coder); + } + } + + private static String newBytesStringSlow(byte coder, byte[] data) { + if (coder == LATIN1) { + return new String(data, StandardCharsets.ISO_8859_1); + } else if (coder == UTF16) { + char[] chars = new char[data.length >> 1]; + for (int i = 0, j = 0; i < data.length; i += 2) { + chars[j++] = (char) ((data[i] & 0xff) | ((data[i + 1] & 0xff) << 8)); + } + return new String(chars); + } else { + return new String(data, StandardCharsets.UTF_8); + } + } + + private static BiFunction getCharsStringZeroCopyCtr() { + if (!STRING_VALUE_FIELD_IS_CHARS) { + return null; + } + MethodHandle handle = getJavaStringZeroCopyCtrHandle(); + if (handle == null) { + return null; + } + try { + CallSite callSite = + LambdaMetafactory.metafactory( + STRING_LOOK_UP, + "apply", + MethodType.methodType(BiFunction.class), + handle.type().generic(), + handle, + handle.type()); + return (BiFunction) callSite.getTarget().invokeExact(); + } catch (Throwable e) { + return null; + } + } + + private static BiFunction getBytesStringZeroCopyCtr() { + if (!STRING_VALUE_FIELD_IS_BYTES) { + return null; + } + MethodHandle handle = getJavaStringZeroCopyCtrHandle(); + if (handle == null) { + return null; + } + try { + MethodType instantiatedMethodType = + MethodType.methodType(handle.type().returnType(), new Class[] {byte[].class, Byte.class}); + CallSite callSite = + LambdaMetafactory.metafactory( + STRING_LOOK_UP, + "apply", + MethodType.methodType(BiFunction.class), + handle.type().generic(), + handle, + instantiatedMethodType); + return (BiFunction) callSite.getTarget().invokeExact(); + } catch (Throwable e) { + return null; + } + } + + private static Function getLatinBytesStringZeroCopyCtr() { + if (!STRING_VALUE_FIELD_IS_BYTES || STRING_LOOK_UP == null) { + return null; + } + try { + Class clazz = Class.forName("java.lang.StringCoding"); + Lookup caller = STRING_LOOK_UP.in(clazz); + MethodHandle handle = + caller.findStatic( + clazz, "newStringLatin1", MethodType.methodType(String.class, byte[].class)); + return makeFunction(caller, handle, Function.class); + } catch (Throwable e) { + return null; + } + } + + private static MethodHandle getJavaStringZeroCopyCtrHandle() { + Preconditions.checkArgument(JdkVersion.MAJOR_VERSION >= 8); + if (STRING_LOOK_UP == null) { + return null; + } + try { + if (STRING_VALUE_FIELD_IS_CHARS) { + return STRING_LOOK_UP.findConstructor( + String.class, MethodType.methodType(void.class, char[].class, boolean.class)); + } else { + return STRING_LOOK_UP.findConstructor( + String.class, MethodType.methodType(void.class, byte[].class, byte.class)); + } + } catch (Exception e) { + return null; + } + } + // Lazy load offsets and keep the access shape in one class so the JDK25 multi-release // replacement can change these methods without touching MemoryUtils callers. private static class ByteArrayStreamFields { @@ -227,6 +381,58 @@ public static void wrap(ByteArrayInputStream stream, MemoryBuffer buffer) { buffer.readerIndex(pos); } + private static class ObjectStreamClassFields { + private static final long WRITE_OBJECT_METHOD; + private static final long READ_OBJECT_METHOD; + private static final long READ_OBJECT_NO_DATA_METHOD; + private static final long WRITE_REPLACE_METHOD; + private static final long READ_RESOLVE_METHOD; + + static { + try { + WRITE_OBJECT_METHOD = + UNSAFE.objectFieldOffset(ObjectStreamClass.class.getDeclaredField("writeObjectMethod")); + READ_OBJECT_METHOD = + UNSAFE.objectFieldOffset(ObjectStreamClass.class.getDeclaredField("readObjectMethod")); + READ_OBJECT_NO_DATA_METHOD = + UNSAFE.objectFieldOffset( + ObjectStreamClass.class.getDeclaredField("readObjectNoDataMethod")); + WRITE_REPLACE_METHOD = + UNSAFE.objectFieldOffset( + ObjectStreamClass.class.getDeclaredField("writeReplaceMethod")); + READ_RESOLVE_METHOD = + UNSAFE.objectFieldOffset(ObjectStreamClass.class.getDeclaredField("readResolveMethod")); + } catch (NoSuchFieldException e) { + throw new RuntimeException(e); + } + } + } + + public static Method getObjectStreamClassWriteObjectMethod(ObjectStreamClass objectStreamClass) { + return (Method) + UNSAFE.getObject(objectStreamClass, ObjectStreamClassFields.WRITE_OBJECT_METHOD); + } + + public static Method getObjectStreamClassReadObjectMethod(ObjectStreamClass objectStreamClass) { + return (Method) UNSAFE.getObject(objectStreamClass, ObjectStreamClassFields.READ_OBJECT_METHOD); + } + + public static Method getObjectStreamClassReadObjectNoDataMethod( + ObjectStreamClass objectStreamClass) { + return (Method) + UNSAFE.getObject(objectStreamClass, ObjectStreamClassFields.READ_OBJECT_NO_DATA_METHOD); + } + + public static Method getObjectStreamClassWriteReplaceMethod(ObjectStreamClass objectStreamClass) { + return (Method) + UNSAFE.getObject(objectStreamClass, ObjectStreamClassFields.WRITE_REPLACE_METHOD); + } + + public static Method getObjectStreamClassReadResolveMethod(ObjectStreamClass objectStreamClass) { + return (Method) + UNSAFE.getObject(objectStreamClass, ObjectStreamClassFields.READ_RESOLVE_METHOD); + } + public static T tryMakeFunction( Lookup lookup, MethodHandle handle, Class functionInterface) { try { @@ -456,4 +662,8 @@ public static Object addReads(Object thisModule, Object otherModule) { throw ExceptionUtils.throwException(e); } } + + public static Lookup privateLookupIn(Class targetClass, Lookup caller) { + return _Lookup.privateLookupIn(targetClass, caller); + } } diff --git a/java/fory-core/src/main/java/org/apache/fory/reflect/FieldAccessor.java b/java/fory-core/src/main/java/org/apache/fory/reflect/FieldAccessor.java index b4919e5da2..2eb540cb79 100644 --- a/java/fory-core/src/main/java/org/apache/fory/reflect/FieldAccessor.java +++ b/java/fory-core/src/main/java/org/apache/fory/reflect/FieldAccessor.java @@ -51,24 +51,68 @@ /** Field accessor for primitive types and object types. */ @SuppressWarnings({"unchecked", "rawtypes"}) public abstract class FieldAccessor { + private static final int REFLECTIVE_ACCESS = 0; + private static final int BOOLEAN_ACCESS = 1; + private static final int BYTE_ACCESS = 2; + private static final int CHAR_ACCESS = 3; + private static final int SHORT_ACCESS = 4; + private static final int INT_ACCESS = 5; + private static final int LONG_ACCESS = 6; + private static final int FLOAT_ACCESS = 7; + private static final int DOUBLE_ACCESS = 8; + private static final int OBJECT_ACCESS = 9; + protected final Field field; protected final long fieldOffset; + private final int accessKind; public FieldAccessor(Field field) { this.field = field; Preconditions.checkNotNull(field); - long fieldOffset; - try { - fieldOffset = ReflectionUtils.getFieldOffset(field); - } catch (UnsupportedOperationException e) { - fieldOffset = -1; + this.fieldOffset = fieldOffset(field); + this.accessKind = accessKind(field, fieldOffset); + } + + private static long fieldOffset(Field field) { + if (AndroidSupport.IS_ANDROID) { + return -1; } - this.fieldOffset = fieldOffset; + if (GraalvmSupport.isGraalBuildTime()) { + // Field offsets are rewritten by GraalVM and are not stable during native-image build time. + return -1; + } + return UnsafeOps.objectFieldOffset(field); } protected FieldAccessor(Field field, long fieldOffset) { this.field = field; this.fieldOffset = fieldOffset; + this.accessKind = accessKind(field, fieldOffset); + } + + private static int accessKind(Field field, long fieldOffset) { + if (fieldOffset == -1) { + return REFLECTIVE_ACCESS; + } + Class fieldType = field.getType(); + if (fieldType == boolean.class) { + return BOOLEAN_ACCESS; + } else if (fieldType == byte.class) { + return BYTE_ACCESS; + } else if (fieldType == char.class) { + return CHAR_ACCESS; + } else if (fieldType == short.class) { + return SHORT_ACCESS; + } else if (fieldType == int.class) { + return INT_ACCESS; + } else if (fieldType == long.class) { + return LONG_ACCESS; + } else if (fieldType == float.class) { + return FLOAT_ACCESS; + } else if (fieldType == double.class) { + return DOUBLE_ACCESS; + } + return OBJECT_ACCESS; } public abstract Object get(Object obj); @@ -77,6 +121,54 @@ public void set(Object obj, Object value) { throw new UnsupportedOperationException("Unsupported for field " + field); } + public final void copy(Object sourceObject, Object targetObject) { + switch (accessKind) { + case BOOLEAN_ACCESS: + UnsafeOps.putBoolean( + targetObject, fieldOffset, UnsafeOps.getBoolean(sourceObject, fieldOffset)); + return; + case BYTE_ACCESS: + UnsafeOps.putByte(targetObject, fieldOffset, UnsafeOps.getByte(sourceObject, fieldOffset)); + return; + case CHAR_ACCESS: + UnsafeOps.putChar(targetObject, fieldOffset, UnsafeOps.getChar(sourceObject, fieldOffset)); + return; + case SHORT_ACCESS: + UnsafeOps.putShort( + targetObject, fieldOffset, UnsafeOps.getShort(sourceObject, fieldOffset)); + return; + case INT_ACCESS: + UnsafeOps.putInt(targetObject, fieldOffset, UnsafeOps.getInt(sourceObject, fieldOffset)); + return; + case LONG_ACCESS: + UnsafeOps.putLong(targetObject, fieldOffset, UnsafeOps.getLong(sourceObject, fieldOffset)); + return; + case FLOAT_ACCESS: + UnsafeOps.putFloat( + targetObject, fieldOffset, UnsafeOps.getFloat(sourceObject, fieldOffset)); + return; + case DOUBLE_ACCESS: + UnsafeOps.putDouble( + targetObject, fieldOffset, UnsafeOps.getDouble(sourceObject, fieldOffset)); + return; + case OBJECT_ACCESS: + UnsafeOps.putObject( + targetObject, fieldOffset, UnsafeOps.getObject(sourceObject, fieldOffset)); + return; + default: + putObject(targetObject, getObject(sourceObject)); + } + } + + public final void copyObject(Object sourceObject, Object targetObject) { + if (accessKind == OBJECT_ACCESS) { + UnsafeOps.putObject( + targetObject, fieldOffset, UnsafeOps.getObject(sourceObject, fieldOffset)); + } else { + putObject(targetObject, getObject(sourceObject)); + } + } + public Field getField() { return field; } @@ -145,7 +237,7 @@ public void putDouble(Object targetObject, double value) { set(targetObject, value); } - public void putObject(Object targetObject, Object object) { + public final void putObject(Object targetObject, Object object) { // For primitive fields, we must use set() which calls the correct UnsafeOps.putXxx method. // UnsafeOps.putObject writes object references, not primitive values. if (fieldOffset != -1 && !field.getType().isPrimitive()) { @@ -155,7 +247,7 @@ public void putObject(Object targetObject, Object object) { } } - public Object getObject(Object targetObject) { + public final Object getObject(Object targetObject) { // For primitive fields, we must use get() which calls the correct UnsafeOps.getXxx method // and returns the boxed value. UnsafeOps.getObject interprets primitive bytes as object // refs. diff --git a/java/fory-core/src/main/java/org/apache/fory/reflect/ObjectCreator.java b/java/fory-core/src/main/java/org/apache/fory/reflect/ObjectCreator.java index cc6d0c7d70..75f8c64558 100644 --- a/java/fory-core/src/main/java/org/apache/fory/reflect/ObjectCreator.java +++ b/java/fory-core/src/main/java/org/apache/fory/reflect/ObjectCreator.java @@ -37,6 +37,10 @@ */ @ThreadSafe public abstract class ObjectCreator { + private static final String[] NO_FIELDS = new String[0]; + private static final Class[] NO_TYPES = new Class[0]; + private static final boolean[] NO_FINAL_FIELDS = new boolean[0]; + protected final Class type; protected ObjectCreator(Class type) { @@ -52,6 +56,34 @@ protected ObjectCreator(Class type) { */ public abstract T newInstance(); + public boolean hasConstructorFields() { + return false; + } + + public String[] getConstructorFieldNames() { + return NO_FIELDS; + } + + public Class[] getConstructorFieldDeclaringClasses() { + return null; + } + + public Class[] getConstructorFieldTypes() { + return NO_TYPES; + } + + public boolean[] getConstructorFieldFinal() { + return NO_FINAL_FIELDS; + } + + public boolean isConstructorPublic() { + return false; + } + + public boolean isOnlyPublicConstructor() { + return false; + } + /** * Creates a new instance of type T using the provided arguments. * diff --git a/java/fory-core/src/main/java/org/apache/fory/reflect/ObjectCreators.java b/java/fory-core/src/main/java/org/apache/fory/reflect/ObjectCreators.java index 065a82b237..a9aa8d2da9 100644 --- a/java/fory-core/src/main/java/org/apache/fory/reflect/ObjectCreators.java +++ b/java/fory-core/src/main/java/org/apache/fory/reflect/ObjectCreators.java @@ -19,10 +19,22 @@ package org.apache.fory.reflect; +import java.lang.annotation.Annotation; import java.lang.invoke.MethodHandle; import java.lang.invoke.MethodHandles.Lookup; import java.lang.invoke.MethodType; import java.lang.reflect.Constructor; +import java.lang.reflect.Field; +import java.lang.reflect.Modifier; +import java.lang.reflect.Parameter; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.apache.fory.annotation.ForyField; import org.apache.fory.collection.ClassValueCache; import org.apache.fory.collection.Tuple2; import org.apache.fory.exception.ForyException; @@ -31,6 +43,8 @@ import org.apache.fory.platform.JdkVersion; import org.apache.fory.platform.UnsafeOps; import org.apache.fory.platform.internal._JDKAccess; +import org.apache.fory.type.Descriptor; +import org.apache.fory.type.TypeUtils; import org.apache.fory.util.record.RecordUtils; /** @@ -90,7 +104,14 @@ private static ObjectCreator creategetObjectCreator(Class type) { if (noArgConstructor != null) { return new ReflectiveNoArgCtrObjectCreator<>(type, noArgConstructor); } - return new UnsupportedObjectCreator<>(type); + return new UnsupportedObjectCreator<>( + type, "Android cannot create " + type + " without an accessible no-arg constructor"); + } + if (JdkVersion.MAJOR_VERSION >= 25 && noArgConstructor == null) { + return new ConstructorObjectCreator<>(type); + } + if (JdkVersion.MAJOR_VERSION >= 25 && hasFinalFields(type)) { + return new ConstructorObjectCreator<>(type); } if (GraalvmSupport.IN_GRAALVM_NATIVE_IMAGE) { if (noArgConstructor != null) { @@ -105,6 +126,328 @@ private static ObjectCreator creategetObjectCreator(Class type) { return new DeclaredNoArgCtrObjectCreator<>(type); } + private static boolean hasFinalFields(Class type) { + for (Field field : Descriptor.getFields(type)) { + if (Modifier.isFinal(field.getModifiers())) { + return true; + } + } + return false; + } + + public static boolean supportsJdk25Creation(Class type) { + if (JdkVersion.MAJOR_VERSION < 25 || RecordUtils.isRecord(type)) { + return true; + } + try { + ObjectCreator creator = creategetObjectCreator(type); + return !(creator instanceof UnsupportedObjectCreator); + } catch (RuntimeException e) { + return false; + } + } + + private static final class ConstructorMatch { + private final Constructor constructor; + private final String[] fieldNames; + private final Class[] declaringClasses; + private final Class[] fieldTypes; + private final boolean[] finalFields; + private final int score; + + private ConstructorMatch( + Constructor constructor, + String[] fieldNames, + Class[] declaringClasses, + Class[] fieldTypes, + boolean[] finalFields, + int score) { + this.constructor = constructor; + this.fieldNames = fieldNames; + this.declaringClasses = declaringClasses; + this.fieldTypes = fieldTypes; + this.finalFields = finalFields; + this.score = score; + } + } + + private static ConstructorMatch findConstructor(Class type) { + List fields = new ArrayList<>(); + fields.addAll(Descriptor.getFields(type)); + Map fieldsByName = new LinkedHashMap<>(); + Map> fieldsByNameList = new LinkedHashMap<>(); + Map fieldsById = new LinkedHashMap<>(); + Set duplicateNames = new LinkedHashSet<>(); + Set duplicateIds = new LinkedHashSet<>(); + Set finalFields = new LinkedHashSet<>(); + for (Field field : fields) { + fieldsByNameList.computeIfAbsent(field.getName(), name -> new ArrayList<>()).add(field); + Field previous = fieldsByName.put(field.getName(), field); + if (previous != null) { + duplicateNames.add(field.getName()); + } + ForyField foryField = field.getAnnotation(ForyField.class); + if (foryField != null && foryField.id() >= 0) { + previous = fieldsById.put(foryField.id(), field); + if (previous != null) { + duplicateIds.add(foryField.id()); + } + } + if (Modifier.isFinal(field.getModifiers())) { + finalFields.add(field); + } + } + ConstructorMatch best = null; + for (Constructor constructor : type.getDeclaredConstructors()) { + if (constructor.isSynthetic()) { + continue; + } + ConstructorMatch match = + matchConstructor( + type, + (Constructor) constructor, + fieldsByName, + fieldsByNameList, + fieldsById, + duplicateNames, + duplicateIds, + finalFields); + if (match != null && (best == null || match.score > best.score)) { + best = match; + } + } + if (best == null) { + String requirement = + finalFields.isEmpty() + ? "a bindable constructor because no no-arg constructor is available" + : "a constructor covering final fields " + finalFields; + throw new ForyException( + "JDK25 zero-Unsafe mode requires " + + requirement + + " for " + + type + + ". Annotate the constructor with java.beans.ConstructorProperties or compile " + + "the class with -parameters."); + } + return best; + } + + private static ConstructorMatch matchConstructor( + Class type, + Constructor constructor, + Map fieldsByName, + Map> fieldsByNameList, + Map fieldsById, + Set duplicateNames, + Set duplicateIds, + Set finalFields) { + Field[] fields = + constructorFields( + constructor, fieldsByName, fieldsByNameList, fieldsById, duplicateNames, duplicateIds); + if (fields == null) { + return null; + } + return matchConstructorFields(constructor, finalFields, fields); + } + + private static ConstructorMatch matchConstructorFields( + Constructor constructor, Set finalFields, Field[] fields) { + if (!containsAllFinalFields(finalFields, fields)) { + return null; + } + Class[] parameterTypes = constructor.getParameterTypes(); + String[] names = new String[fields.length]; + Class[] declaringClasses = new Class[fields.length]; + Class[] fieldTypes = new Class[fields.length]; + boolean[] finalFieldFlags = new boolean[fields.length]; + for (int i = 0; i < fields.length; i++) { + Field field = fields[i]; + if (!constructorTypeMatches(parameterTypes[i], field)) { + return null; + } + names[i] = field.getName(); + declaringClasses[i] = field.getDeclaringClass(); + fieldTypes[i] = field.getType(); + finalFieldFlags[i] = Modifier.isFinal(field.getModifiers()); + } + return new ConstructorMatch<>( + constructor, names, declaringClasses, fieldTypes, finalFieldFlags, 300 - fields.length); + } + + private static boolean containsAllFinalFields(Set finalFields, Field[] fields) { + Set selectedFields = new LinkedHashSet<>(Arrays.asList(fields)); + for (Field finalField : finalFields) { + if (selectedFields.contains(finalField)) { + continue; + } + if (coveredBySyntheticField(finalField, selectedFields)) { + continue; + } + return false; + } + return true; + } + + private static boolean coveredBySyntheticField(Field finalField, Set selectedFields) { + if (!finalField.isSynthetic()) { + return false; + } + for (Field selectedField : selectedFields) { + if (selectedField.isSynthetic() + && selectedField.getName().equals(finalField.getName()) + && selectedField.getType() == finalField.getType()) { + return true; + } + } + return false; + } + + private static Field[] constructorFields( + Constructor constructor, + Map fieldsByName, + Map> fieldsByNameList, + Map fieldsById, + Set duplicateNames, + Set duplicateIds) { + Field[] fields = fieldsByForyFieldId(constructor, fieldsById, duplicateIds); + if (fields != null) { + return fields; + } + String[] names = constructorFieldNames(constructor); + if (names != null) { + if (names.length != constructor.getParameterCount()) { + return null; + } + return fieldsByName(constructor, fieldsByName, fieldsByNameList, duplicateNames, names); + } + return null; + } + + private static Field[] fieldsByForyFieldId( + Constructor constructor, Map fieldsById, Set duplicateIds) { + Parameter[] parameters = constructor.getParameters(); + Field[] fields = new Field[parameters.length]; + boolean hasForyFieldId = false; + for (int i = 0; i < parameters.length; i++) { + ForyField foryField = parameters[i].getAnnotation(ForyField.class); + if (foryField == null || foryField.id() < 0) { + continue; + } + hasForyFieldId = true; + int id = foryField.id(); + if (duplicateIds.contains(id)) { + throw new ForyException("Constructor parameter id " + id + " is ambiguous"); + } + fields[i] = fieldsById.get(id); + if (fields[i] == null) { + return null; + } + } + if (!hasForyFieldId) { + return null; + } + for (Field field : fields) { + if (field == null) { + return null; + } + } + return fields; + } + + private static Field[] fieldsByName( + Constructor constructor, + Map fieldsByName, + Map> fieldsByNameList, + Set duplicateNames, + String[] names) { + Field[] fields = new Field[names.length]; + for (int i = 0; i < names.length; i++) { + String name = names[i]; + if (duplicateNames.contains(name)) { + Field field = syntheticField(constructor, fieldsByNameList.get(name)); + if (field == null) { + throw new ForyException( + "Constructor parameter " + + name + + " is ambiguous because " + + constructor.getDeclaringClass() + + " has duplicate field names"); + } + fields[i] = field; + continue; + } + Field field = fieldsByName.get(name); + if (field == null) { + return null; + } + fields[i] = field; + } + return fields; + } + + private static Field syntheticField(Constructor constructor, List fields) { + if (fields == null) { + return null; + } + Class declaringClass = constructor.getDeclaringClass(); + for (Field field : fields) { + if (field.isSynthetic() && field.getDeclaringClass() == declaringClass) { + return field; + } + } + return null; + } + + private static boolean constructorTypeMatches(Class parameterType, Field field) { + Class boxedParameterType = TypeUtils.boxedType(parameterType); + Class boxedFieldType = TypeUtils.boxedType(field.getType()); + return boxedParameterType.isAssignableFrom(boxedFieldType); + } + + private static String[] constructorFieldNames(Constructor constructor) { + String[] names = constructorProperties(constructor); + if (names != null) { + return names; + } + Parameter[] parameters = constructor.getParameters(); + for (Parameter parameter : parameters) { + if (!parameter.isNamePresent()) { + return null; + } + } + names = new String[parameters.length]; + for (int i = 0; i < parameters.length; i++) { + names[i] = parameters[i].getName(); + } + return names; + } + + private static String[] constructorProperties(Constructor constructor) { + for (Annotation annotation : constructor.getDeclaredAnnotations()) { + if ("java.beans.ConstructorProperties".equals(annotation.annotationType().getName())) { + try { + return (String[]) annotation.annotationType().getMethod("value").invoke(annotation); + } catch (ReflectiveOperationException e) { + throw new ForyException("Failed to read ConstructorProperties for " + constructor, e); + } + } + } + return null; + } + + private static MethodHandle constructorHandle(Class type, Constructor constructor) { + Lookup lookup = _JDKAccess._trustedLookup(type); + if (lookup == null) { + return null; + } + try { + return lookup.findConstructor( + type, MethodType.methodType(void.class, constructor.getParameterTypes())); + } catch (NoSuchMethodException | IllegalAccessException e) { + return null; + } + } + private static final class ReflectiveNoArgCtrObjectCreator extends ObjectCreator { private final Constructor constructor; @@ -134,20 +477,109 @@ public T newInstanceWithArguments(Object... arguments) { } private static final class UnsupportedObjectCreator extends ObjectCreator { - private UnsupportedObjectCreator(Class type) { + private final String message; + + private UnsupportedObjectCreator(Class type, String message) { super(type); + this.message = message; } @Override public T newInstance() { - throw new ForyException( - "Android cannot create " + type + " without an accessible no-arg constructor"); + throw new ForyException(message); } @Override public T newInstanceWithArguments(Object... arguments) { + throw new ForyException(message); + } + } + + public static final class ConstructorObjectCreator extends ObjectCreator { + private final Constructor constructor; + private final MethodHandle handle; + private final String[] fieldNames; + private final Class[] declaringClasses; + private final Class[] fieldTypes; + private final boolean[] finalFields; + + private ConstructorObjectCreator(Class type) { + super(type); + ConstructorMatch match = findConstructor(type); + constructor = match.constructor; + handle = constructorHandle(type, constructor); + fieldNames = match.fieldNames; + declaringClasses = match.declaringClasses; + fieldTypes = match.fieldTypes; + finalFields = match.finalFields; + try { + constructor.setAccessible(true); + } catch (RuntimeException e) { + throw new ForyException("Failed to make constructor accessible for " + type, e); + } + } + + @Override + public boolean hasConstructorFields() { + return true; + } + + @Override + public String[] getConstructorFieldNames() { + return fieldNames; + } + + @Override + public Class[] getConstructorFieldDeclaringClasses() { + return declaringClasses; + } + + @Override + public Class[] getConstructorFieldTypes() { + return fieldTypes; + } + + @Override + public boolean[] getConstructorFieldFinal() { + return finalFields; + } + + @Override + public boolean isConstructorPublic() { + return Modifier.isPublic(type.getModifiers()) + && Modifier.isPublic(constructor.getModifiers()); + } + + @Override + public boolean isOnlyPublicConstructor() { + if (!isConstructorPublic()) { + return false; + } + for (Constructor declaredConstructor : type.getDeclaredConstructors()) { + if (Modifier.isPublic(declaredConstructor.getModifiers()) + && declaredConstructor != constructor) { + return false; + } + } + return true; + } + + @Override + public T newInstance() { throw new ForyException( - "Android cannot create " + type + " without a supported constructor path"); + "JDK25 zero-Unsafe mode requires constructor field values to create " + type); + } + + @Override + public T newInstanceWithArguments(Object... arguments) { + try { + if (handle == null) { + return constructor.newInstance(arguments); + } + return (T) handle.invokeWithArguments(arguments); + } catch (Throwable e) { + throw new ForyException("Failed to create instance using constructor: " + type, e); + } } } diff --git a/java/fory-core/src/main/java/org/apache/fory/reflect/ReflectionUtils.java b/java/fory-core/src/main/java/org/apache/fory/reflect/ReflectionUtils.java index de19e8e262..05f0e23712 100644 --- a/java/fory-core/src/main/java/org/apache/fory/reflect/ReflectionUtils.java +++ b/java/fory-core/src/main/java/org/apache/fory/reflect/ReflectionUtils.java @@ -51,8 +51,6 @@ import org.apache.fory.collection.ClassValueCache; import org.apache.fory.exception.ForyException; import org.apache.fory.platform.AndroidSupport; -import org.apache.fory.platform.GraalvmSupport; -import org.apache.fory.platform.UnsafeOps; import org.apache.fory.platform.internal._JDKAccess; import org.apache.fory.util.ExceptionUtils; import org.apache.fory.util.Preconditions; @@ -463,35 +461,6 @@ public static List getFieldValues(Collection fields, Object o) { return results; } - public static long getFieldOffset(Field field) { - if (AndroidSupport.IS_ANDROID) { - throw new UnsupportedOperationException( - "Field offsets are not supported on Android: " + field); - } - if (GraalvmSupport.isGraalBuildTime()) { - // See more details at - // https://www.graalvm.org/latest/reference-manual/native-image/metadata/Compatibility/#unsafe-memory-access - throw new IllegalStateException( - "Field offset will change between graalvm build time and runtime, " - + "should bye accessed by following graalvm auto rewrite pattern."); - } - if (field == null) { - return -1; - } - return UnsafeOps.objectFieldOffset(field); - } - - public static long getFieldOffset(Class cls, String fieldName) { - Field field = getFieldNullable(cls, fieldName); - return getFieldOffset(field); - } - - public static long getFieldOffsetChecked(Class cls, String fieldName) { - long offset = getFieldOffset(cls, fieldName); - Preconditions.checkArgument(offset != -1); - return offset; - } - public static void setObjectFieldValue(Object obj, String fieldName, Object value) { setObjectFieldValue(obj, getField(obj.getClass(), fieldName), value); } diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java index 49c4a6143f..24f3d7f472 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java @@ -31,8 +31,10 @@ import java.io.IOException; import java.io.Serializable; import java.lang.invoke.SerializedLambda; +import java.lang.reflect.Modifier; import java.math.BigDecimal; import java.math.BigInteger; +import java.net.URL; import java.nio.ByteBuffer; import java.nio.charset.Charset; import java.time.Duration; @@ -57,6 +59,7 @@ import java.util.Objects; import java.util.Optional; import java.util.OptionalInt; +import java.util.StringTokenizer; import java.util.TimeZone; import java.util.TreeMap; import java.util.TreeSet; @@ -100,6 +103,7 @@ import org.apache.fory.logging.LoggerFactory; import org.apache.fory.memory.ByteBufferUtil; import org.apache.fory.memory.MemoryBuffer; +import org.apache.fory.memory.MemoryUtils; import org.apache.fory.meta.ClassSpec; import org.apache.fory.meta.EncodedMetaString; import org.apache.fory.meta.Encoders; @@ -108,11 +112,13 @@ import org.apache.fory.meta.TypeExtMeta; import org.apache.fory.platform.AndroidSupport; import org.apache.fory.platform.GraalvmSupport; +import org.apache.fory.platform.JdkVersion; import org.apache.fory.reflect.ObjectCreators; import org.apache.fory.reflect.ReflectionUtils; import org.apache.fory.serializer.ArraySerializers; import org.apache.fory.serializer.BufferSerializers; import org.apache.fory.serializer.CodegenSerializer.LazyInitBeanSerializer; +import org.apache.fory.serializer.CompatibleSerializer; import org.apache.fory.serializer.CopyOnlyObjectSerializer; import org.apache.fory.serializer.EnumSerializer; import org.apache.fory.serializer.ExceptionSerializers; @@ -134,6 +140,7 @@ import org.apache.fory.serializer.Shareable; import org.apache.fory.serializer.SqlTimeSerializers; import org.apache.fory.serializer.TimeSerializers; +import org.apache.fory.serializer.URLSerializer; import org.apache.fory.serializer.UnknownClass; import org.apache.fory.serializer.UnknownClass.UnknownEmptyStruct; import org.apache.fory.serializer.UnknownClass.UnknownStruct; @@ -997,6 +1004,7 @@ private boolean usesNonStructTypeDef(Class cls) { || isMap(cls) || Externalizable.class.isAssignableFrom(cls) || requireJavaSerialization(cls) + || requiresJavaSerializer(cls) || useReplaceResolveSerializer(cls) || Functions.isLambda(cls) || (ScalaTypes.SCALA_AVAILABLE && ReflectionUtils.isScalaSingletonObject(cls)) @@ -1467,6 +1475,15 @@ public Class getSerializerClass(Class cls, boolean code return TimeSerializers.ZoneIdSerializer.class; } else if (TimeZone.class.isAssignableFrom(cls)) { return TimeSerializers.TimeZoneSerializer.class; + } else if (cls == URL.class) { + return URLSerializer.class; + } else if (cls == StringTokenizer.class) { + if (!MemoryUtils.JDK_COLLECTION_FIELD_ACCESS) { + throw new UnsupportedOperationException( + "StringTokenizer serialization requires JDK internal field access. On JDK25+, open " + + "java.base/java.util to org.apache.fory.core,org.apache.fory.format."); + } + return Serializers.StringTokenizerSerializer.class; } else if (ByteBuffer.class.isAssignableFrom(cls)) { return BufferSerializers.ByteBufferSerializer.class; } @@ -1526,6 +1543,12 @@ public Class getSerializerClass(Class cls, boolean code return MapSerializer.class; } } + if (requiresJdkStream(cls)) { + return getDefaultJDKStreamSerializerType(); + } + if (requiresJavaSerializer(cls)) { + return JavaSerializer.class; + } if (isCrossLanguage()) { LOG.warn("Class {} isn't supported for cross-language serialization.", cls); } @@ -1564,6 +1587,9 @@ public Object id() { public Class getObjectSerializerClass( Class cls, JITContext.SerializerJITCallback> callback) { boolean codegen = config.isCodeGenEnabled() && supportCodegenForJavaSerialization(cls); + if (JdkVersion.MAJOR_VERSION >= 25 && !Modifier.isPublic(cls.getModifiers())) { + codegen = false; + } return getObjectSerializerClass(cls, false, codegen, callback); } @@ -1579,6 +1605,9 @@ public Class getObjectSerializerClass( return serializerClass; } } + if (ReflectionUtils.isJdkProxy(cls)) { + return JdkProxySerializer.class; + } Class staticSerializerClass = getStaticGeneratedStructSerializerClass(cls); if (staticSerializerClass != null && shouldPreferStaticGeneratedSerializer(cls)) { @@ -1610,6 +1639,34 @@ public Class getObjectSerializerClass( } } + private static boolean requiresJdkStream(Class cls) { + return JdkVersion.MAJOR_VERSION >= 25 + && cls.getName().startsWith("java.") + && Serializable.class.isAssignableFrom(cls) + && !hasNoArgConstructor(cls); + } + + private static boolean requiresJavaSerializer(Class cls) { + if (JdkVersion.MAJOR_VERSION < 25 || !Serializable.class.isAssignableFrom(cls)) { + return false; + } + // Scala products can have final derived fields initialized by the primary constructor but not + // represented as constructor parameters. Keep that compatibility in the isolated JDK stream + // path instead of teaching the generic JDK25 field serializer to ignore final fields. + return ScalaTypes.SCALA_AVAILABLE + && ScalaTypes.isScalaProductType(cls) + && !ReflectionUtils.isScalaSingletonObject(cls); + } + + private static boolean hasNoArgConstructor(Class cls) { + try { + cls.getDeclaredConstructor(); + return true; + } catch (NoSuchMethodException e) { + return false; + } + } + public Class getJavaSerializer(Class clz) { if (Collection.class.isAssignableFrom(clz)) { return CollectionSerializers.JDKCompatibleCollectionSerializer.class; @@ -1860,7 +1917,21 @@ private void registerGraalvmSerializerClass(Class cls) { RecordUtils.getRecordConstructor(cls); RecordUtils.getRecordComponents(cls); } - ObjectCreators.getObjectCreator(cls); + if (needsGraalvmObjectCreator(cls, serializerClass)) { + ObjectCreators.getObjectCreator(cls); + } + } + + private boolean needsGraalvmObjectCreator( + Class cls, Class serializerClass) { + if (cls.isArray()) { + return false; + } + return serializerClass == ObjectSerializer.class + || serializerClass == CompatibleSerializer.class + || serializerClass == ReplaceResolveSerializer.class + || serializerClass == CollectionSerializers.DefaultJavaCollectionSerializer.class + || serializerClass == MapSerializers.DefaultJavaMapSerializer.class; } private void createSerializer0(Class cls) { diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/AbstractObjectSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/AbstractObjectSerializer.java index 58bc955f9d..8b8dc6376b 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/AbstractObjectSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/AbstractObjectSerializer.java @@ -32,6 +32,7 @@ import org.apache.fory.context.RefReader; import org.apache.fory.context.RefWriter; import org.apache.fory.context.WriteContext; +import org.apache.fory.exception.ForyException; import org.apache.fory.logging.Logger; import org.apache.fory.logging.LoggerFactory; import org.apache.fory.memory.MemoryBuffer; @@ -62,6 +63,7 @@ public abstract class AbstractObjectSerializer extends Serializer { private static final Logger LOG = LoggerFactory.getLogger(AbstractObjectSerializer.class); + private static final Object SELF_REFERENCE = new Object(); protected final Config config; protected final TypeResolver typeResolver; protected final boolean isRecord; @@ -863,6 +865,9 @@ public T copy(CopyContext copyContext, T originObj) { if (isRecord) { return copyRecord(copyContext, originObj); } + if (objectCreator.hasConstructorFields()) { + return copyConstructorObject(copyContext, originObj); + } T newObj = newBean(); copyContext.reference(originObj, newObj); copyFields(copyContext, originObj, newObj); @@ -870,7 +875,8 @@ public T copy(CopyContext copyContext, T originObj) { } private T copyRecord(CopyContext copyContext, T originObj) { - Object[] fieldValues = copyFields(copyContext, originObj); + Object[] fieldValues = copyFieldValues(copyContext, originObj); + fieldValues = RecordUtils.remapping(copyRecordInfo, fieldValues); try { T t = objectCreator.newInstanceWithArguments(fieldValues); Arrays.fill(copyRecordInfo.getRecordComponents(), null); @@ -882,7 +888,32 @@ private T copyRecord(CopyContext copyContext, T originObj) { return originObj; } - private Object[] copyFields(CopyContext copyContext, T originObj) { + private T copyConstructorObject(CopyContext copyContext, T originObj) { + SerializationFieldInfo[] fieldInfos = this.fieldInfos; + if (fieldInfos == null) { + fieldInfos = buildFieldsInfo(); + } + int[] constructorFieldIndexes = buildConstructorFieldIndexes(fieldInfos); + boolean[] constructorFieldMask = + buildConstructorFieldMask(fieldInfos.length, constructorFieldIndexes); + copyContext.markCopying(originObj); + try { + Object[] fieldValues = + copyFieldValues(copyContext, originObj, fieldInfos, constructorFieldMask, true); + T newObj = + objectCreator.newInstanceWithArguments( + constructorArgs( + fieldValues, constructorFieldIndexes, objectCreator.getConstructorFieldTypes())); + copyContext.reference(originObj, newObj); + copyFields(copyContext, fieldInfos, originObj, newObj, constructorFieldMask, false); + return newObj; + } catch (Throwable e) { + copyContext.cancelCopy(originObj); + throw e; + } + } + + private Object[] copyFieldValues(CopyContext copyContext, T originObj) { SerializationFieldInfo[] fieldInfos = this.fieldInfos; if (fieldInfos == null) { fieldInfos = buildFieldsInfo(); @@ -898,7 +929,30 @@ private Object[] copyFields(CopyContext copyContext, T originObj) { copyNotPrimitiveField(copyContext, originObj, fieldAccessor, fieldInfo.dispatchId); } } - return RecordUtils.remapping(copyRecordInfo, fieldValues); + return fieldValues; + } + + private Object[] copyFieldValues( + CopyContext copyContext, + T originObj, + SerializationFieldInfo[] fieldInfos, + boolean[] constructorFieldMask, + boolean constructorFields) { + Object[] fieldValues = new Object[fieldInfos.length]; + for (int i = 0; i < fieldInfos.length; i++) { + if (constructorFieldMask[i] != constructorFields) { + continue; + } + SerializationFieldInfo fieldInfo = fieldInfos[i]; + FieldAccessor fieldAccessor = fieldInfo.fieldAccessor; + if (fieldInfo.isPrimitiveField) { + fieldValues[i] = copyPrimitiveField(originObj, fieldAccessor, fieldInfo.dispatchId); + } else { + fieldValues[i] = + copyNotPrimitiveField(copyContext, originObj, fieldAccessor, fieldInfo.dispatchId); + } + } + return fieldValues; } private void copyFields(CopyContext copyContext, T originObj, T newObj) { @@ -925,10 +979,36 @@ public static void copyFields( } } + private static void copyFields( + CopyContext copyContext, + SerializationFieldInfo[] fieldInfos, + Object originObj, + Object newObj, + boolean[] constructorFieldMask, + boolean constructorFields) { + for (int i = 0; i < fieldInfos.length; i++) { + if (constructorFieldMask[i] != constructorFields) { + continue; + } + SerializationFieldInfo fieldInfo = fieldInfos[i]; + FieldAccessor fieldAccessor = fieldInfo.fieldAccessor; + if (fieldInfo.isPrimitiveField) { + copySetPrimitiveField(originObj, newObj, fieldAccessor, fieldInfo.dispatchId); + } else { + copySetNotPrimitiveField( + copyContext, originObj, newObj, fieldAccessor, fieldInfo.dispatchId); + } + } + } + private static Object copyFieldValue(CopyContext copyContext, Object fieldValue, int dispatchId) { if (fieldValue == null) { return null; } + return isCopyByReference(dispatchId) ? fieldValue : copyContext.copyObject(fieldValue); + } + + private static boolean isCopyByReference(int dispatchId) { switch (dispatchId) { case DispatchId.BOOL: case DispatchId.INT8: @@ -957,58 +1037,15 @@ private static Object copyFieldValue(CopyContext copyContext, Object fieldValue, case DispatchId.FLOAT16: case DispatchId.BFLOAT16: case DispatchId.STRING: - return fieldValue; + return true; default: - return copyContext.copyObject(fieldValue); + return false; } } private static void copySetPrimitiveField( Object originObj, Object newObj, FieldAccessor fieldAccessor, int typeId) { - switch (typeId) { - case DispatchId.BOOL: - fieldAccessor.putBoolean(newObj, fieldAccessor.getBoolean(originObj)); - break; - case DispatchId.INT8: - fieldAccessor.putByte(newObj, fieldAccessor.getByte(originObj)); - break; - case DispatchId.UINT8: - fieldAccessor.putInt(newObj, fieldAccessor.getInt(originObj)); - break; - case DispatchId.CHAR: - fieldAccessor.putChar(newObj, fieldAccessor.getChar(originObj)); - break; - case DispatchId.INT16: - fieldAccessor.putShort(newObj, fieldAccessor.getShort(originObj)); - break; - case DispatchId.UINT16: - fieldAccessor.putInt(newObj, fieldAccessor.getInt(originObj)); - break; - case DispatchId.INT32: - case DispatchId.VARINT32: - fieldAccessor.putInt(newObj, fieldAccessor.getInt(originObj)); - break; - case DispatchId.UINT32: - case DispatchId.VAR_UINT32: - fieldAccessor.putLong(newObj, fieldAccessor.getLong(originObj)); - break; - case DispatchId.INT64: - case DispatchId.VARINT64: - case DispatchId.TAGGED_INT64: - case DispatchId.UINT64: - case DispatchId.VAR_UINT64: - case DispatchId.TAGGED_UINT64: - fieldAccessor.putLong(newObj, fieldAccessor.getLong(originObj)); - break; - case DispatchId.FLOAT32: - fieldAccessor.putFloat(newObj, fieldAccessor.getFloat(originObj)); - break; - case DispatchId.FLOAT64: - fieldAccessor.putDouble(newObj, fieldAccessor.getDouble(originObj)); - break; - default: - throw new RuntimeException("Unknown primitive type: " + typeId); - } + fieldAccessor.copy(originObj, newObj); } private static void copySetNotPrimitiveField( @@ -1017,8 +1054,12 @@ private static void copySetNotPrimitiveField( Object newObj, FieldAccessor fieldAccessor, int typeId) { + if (isCopyByReference(typeId)) { + fieldAccessor.copyObject(originObj, newObj); + return; + } Object fieldValue = fieldAccessor.getObject(originObj); - fieldAccessor.putObject(newObj, copyFieldValue(copyContext, fieldValue, typeId)); + fieldAccessor.putObject(newObj, fieldValue == null ? null : copyContext.copyObject(fieldValue)); } private Object copyPrimitiveField(Object targetObject, FieldAccessor fieldAccessor, int typeId) { @@ -1108,4 +1149,224 @@ private SerializationFieldInfo[] buildFieldsInfo() { protected T newBean() { return objectCreator.newInstance(); } + + protected final int[] buildConstructorFieldIndexes(SerializationFieldInfo[] fieldInfos) { + return buildConstructorFieldIndexes(fieldInfos, true); + } + + protected final int[] buildConstructorFieldIndexes( + SerializationFieldInfo[] fieldInfos, boolean allowMissingNonFinal) { + return buildConstructorFieldIndexes(fieldInfos, allowMissingNonFinal, null); + } + + protected final int[] buildConstructorFieldIndexes( + SerializationFieldInfo[] fieldInfos, boolean allowMissingNonFinal, String[] defaultFields) { + return buildConstructorFieldIndexes(fieldInfos, allowMissingNonFinal, defaultFields, null); + } + + protected final int[] buildConstructorFieldIndexes( + SerializationFieldInfo[] fieldInfos, + boolean allowMissingNonFinal, + String[] defaultFields, + Class[] defaultDeclaringClasses) { + String[] fieldNames = objectCreator.getConstructorFieldNames(); + if (fieldNames.length == 0) { + return null; + } + Class[] declaringClasses = objectCreator.getConstructorFieldDeclaringClasses(); + boolean[] finalFields = objectCreator.getConstructorFieldFinal(); + int[] indexes = new int[fieldNames.length]; + for (int i = 0; i < fieldNames.length; i++) { + Class declaringClass = declaringClasses == null ? null : declaringClasses[i]; + boolean allowMissing = + (allowMissingNonFinal && !finalFields[i]) + || contains(defaultFields, defaultDeclaringClasses, fieldNames[i], declaringClass); + indexes[i] = constructorFieldIndex(fieldInfos, declaringClass, fieldNames[i], allowMissing); + } + return indexes; + } + + private static boolean contains( + String[] values, Class[] declaringClasses, String value, Class declaringClass) { + if (values == null) { + return false; + } + for (int i = 0; i < values.length; i++) { + if (values[i].equals(value) + && (declaringClasses == null + || i >= declaringClasses.length + || declaringClasses[i] == null + || declaringClasses[i] == declaringClass)) { + return true; + } + } + return false; + } + + protected final boolean[] buildConstructorFieldMask(int size, int[] indexes) { + if (indexes == null) { + return null; + } + boolean[] mask = new boolean[size]; + for (int index : indexes) { + if (index >= 0) { + mask[index] = true; + } + } + return mask; + } + + protected final Object[] constructorArgs(Object[] fieldValues, int[] indexes) { + Object[] args = new Object[indexes.length]; + for (int i = 0; i < indexes.length; i++) { + args[i] = fieldValues[indexes[i]]; + } + return args; + } + + protected final Object[] constructorArgs( + Object[] fieldValues, int[] indexes, Class[] fieldTypes) { + Object[] args = new Object[indexes.length]; + for (int i = 0; i < indexes.length; i++) { + int index = indexes[i]; + args[i] = index < 0 ? defaultConstructorValue(fieldTypes[i]) : fieldValues[index]; + } + return args; + } + + protected final void checkNoUnresolvedReadRef(ReadContext readContext) { + checkNoUnresolvedReadRef(readContext, type); + } + + public static void checkNoUnresolvedReadRef(ReadContext readContext, Class type) { + if (consumeSelfRef(readContext)) { + throwConstructorCycle(type); + } + } + + public static void beginConstructorRef(ReadContext readContext) { + if (readContext.hasPreservedRefId()) { + readContext.trackUnresolvedRef(readContext.lastPreservedRefId()); + } + } + + public static void endConstructorRef(ReadContext readContext) { + readContext.untrackUnresolvedRef(); + } + + public static void referenceConstructorRef(ReadContext readContext, Object object) { + if (readContext.hasTrackedRef()) { + // Constructor-bound objects are registered after some fields may already have reserved and + // resolved their own ids, so bind the tracked id instead of popping the current stack top. + readContext.reference(readContext.currentTrackedRefId(), object); + } else if (readContext.hasPreservedRefId()) { + readContext.reference(object); + } + } + + public static Object ctorFieldValue(ReadContext readContext, Object value, Class type) { + if (consumeSelfRef(readContext)) { + throwConstructorCycle(type); + } + return value; + } + + public static Object bufferFieldValue(ReadContext readContext, Object value, Class type) { + if (!consumeSelfRef(readContext)) { + return value; + } + if (value == null) { + return SELF_REFERENCE; + } + throwConstructorCycle(type); + return null; + } + + public static Object resolveBufferedValue(Object value, Object targetObject) { + return value == SELF_REFERENCE ? targetObject : value; + } + + private static boolean consumeSelfRef(ReadContext readContext) { + if (readContext.hasTrackedRef()) { + return readContext.consumeUnresolvedRef(readContext.currentTrackedRefId()); + } + return readContext.hasPreservedRefId() + && readContext.consumeUnresolvedRef(readContext.lastPreservedRefId()); + } + + private static void throwConstructorCycle(Class type) { + throw new ForyException( + "Cyclic references to constructor-bound type " + + type.getName() + + " cannot be restored before the object is constructed. Use a no-arg constructor " + + "or keep the cycle as a direct non-constructor field."); + } + + public static Object defaultConstructorValue(Class type) { + if (type == boolean.class) { + return false; + } else if (type == byte.class) { + return (byte) 0; + } else if (type == short.class) { + return (short) 0; + } else if (type == char.class) { + return (char) 0; + } else if (type == int.class) { + return 0; + } else if (type == long.class) { + return 0L; + } else if (type == float.class) { + return 0.0f; + } else if (type == double.class) { + return 0.0d; + } + return null; + } + + protected final void setNonConstructorFields( + Object targetObject, + Object[] fieldValues, + SerializationFieldInfo[] fieldInfos, + boolean[] constructorMask) { + for (int i = 0; i < fieldInfos.length; i++) { + if (!constructorMask[i] && fieldInfos[i].fieldAccessor != null) { + fieldInfos[i].fieldAccessor.putObject(targetObject, fieldValues[i]); + } + } + } + + public static Object fieldValue(Object[] fieldValues, int index) { + return fieldValues[index]; + } + + private static int constructorFieldIndex( + SerializationFieldInfo[] fieldInfos, + Class declaringClass, + String fieldName, + boolean allowMissing) { + int index = -1; + for (int i = 0; i < fieldInfos.length; i++) { + FieldAccessor fieldAccessor = fieldInfos[i].fieldAccessor; + if (fieldAccessor == null) { + continue; + } + Field field = fieldAccessor.getField(); + if (!field.getName().equals(fieldName) + || (declaringClass != null && field.getDeclaringClass() != declaringClass)) { + continue; + } + if (index >= 0) { + throw new ForyException( + "Constructor field " + fieldName + " is ambiguous because multiple fields match"); + } + index = i; + } + if (index < 0) { + if (allowMissing) { + return -1; + } + throw new ForyException("Constructor field " + fieldName + " is not serialized"); + } + return index; + } } diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/CompatibleLayerSerializerBase.java b/java/fory-core/src/main/java/org/apache/fory/serializer/CompatibleLayerSerializerBase.java index f7fec4063f..611dfc0da4 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/CompatibleLayerSerializerBase.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/CompatibleLayerSerializerBase.java @@ -29,6 +29,7 @@ import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.meta.TypeDef; import org.apache.fory.reflect.FieldAccessor; +import org.apache.fory.reflect.ObjectCreator; import org.apache.fory.resolver.TypeResolver; import org.apache.fory.serializer.FieldGroups.SerializationFieldInfo; import org.apache.fory.type.DescriptorGrouper; @@ -41,12 +42,34 @@ */ @SuppressWarnings({"unchecked", "rawtypes"}) public abstract class CompatibleLayerSerializerBase extends AbstractObjectSerializer { + private static final ObjectCreator FIELD_ONLY_CREATOR = new FieldOnlyCreator(); + + private static final class FieldOnlyCreator extends ObjectCreator { + private FieldOnlyCreator() { + super(Object.class); + } + + @Override + public Object newInstance() { + throw new UnsupportedOperationException("Layer serializers do not create objects"); + } + + @Override + public Object newInstanceWithArguments(Object... arguments) { + throw new UnsupportedOperationException("Layer serializers do not create objects"); + } + } + protected TypeDef layerTypeDef; protected Class layerMarkerClass; protected SerializationFieldInfo[] allFields = new SerializationFieldInfo[0]; public CompatibleLayerSerializerBase(TypeResolver typeResolver, Class type) { - super(typeResolver, type); + super(typeResolver, type, fieldOnlyCreator()); + } + + private static ObjectCreator fieldOnlyCreator() { + return (ObjectCreator) FIELD_ONLY_CREATOR; } public final void setLayerSerializerMeta(TypeDef layerTypeDef, Class layerMarkerClass) { diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/CompatibleSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/CompatibleSerializer.java index a4f2de53f6..d5e548fb45 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/CompatibleSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/CompatibleSerializer.java @@ -33,6 +33,7 @@ import org.apache.fory.meta.TypeDef; import org.apache.fory.platform.AndroidSupport; import org.apache.fory.platform.GraalvmSupport; +import org.apache.fory.platform.JdkVersion; import org.apache.fory.platform.UnsafeOps; import org.apache.fory.reflect.FieldAccessor; import org.apache.fory.resolver.ClassResolver; @@ -68,6 +69,8 @@ public class CompatibleSerializer extends AbstractObjectSerializer { private static final Logger LOG = LoggerFactory.getLogger(CompatibleSerializer.class); private final SerializationFieldInfo[] allFields; + private final int[] constructorFieldIndexes; + private final boolean[] constructorFieldMask; private final CompatibleCollectionArrayReader.ReadAction[] allCompatibleReadActions; private final boolean hasCompatibleCollectionArrayRead; private final RecordInfo recordInfo; @@ -139,6 +142,34 @@ public CompatibleSerializer(TypeResolver typeResolver, Class type, TypeDef ty } this.hasDefaultValues = hasDefaultValues; this.defaultValueFields = defaultValueFields; + if (!isRecord && objectCreator.hasConstructorFields()) { + constructorFieldIndexes = + buildConstructorFieldIndexes( + allFields, + true, + defaultFieldNames(defaultValueFields), + defaultDeclaringClasses(defaultValueFields)); + constructorFieldMask = buildConstructorFieldMask(allFields.length, constructorFieldIndexes); + } else { + constructorFieldIndexes = null; + constructorFieldMask = null; + } + } + + private static String[] defaultFieldNames(DefaultValueUtils.DefaultValueField[] fields) { + String[] names = new String[fields.length]; + for (int i = 0; i < fields.length; i++) { + names[i] = fields[i].getFieldName(); + } + return names; + } + + private static Class[] defaultDeclaringClasses(DefaultValueUtils.DefaultValueField[] fields) { + Class[] declaringClasses = new Class[fields.length]; + for (int i = 0; i < fields.length; i++) { + declaringClasses[i] = fields[i].getDeclaringClass(); + } + return declaringClasses; } /** Used by generated compatible serializers for top-level list/array compatible field reads. */ @@ -232,7 +263,9 @@ private T newInstance() { return newBean(); } T obj = - AndroidSupport.IS_ANDROID || GraalvmSupport.IN_GRAALVM_NATIVE_IMAGE + AndroidSupport.IS_ANDROID + || GraalvmSupport.IN_GRAALVM_NATIVE_IMAGE + || JdkVersion.MAJOR_VERSION >= 25 ? newBean() : UnsafeOps.newInstance(type); // Set default values for missing fields in Scala case classes @@ -240,6 +273,34 @@ private T newInstance() { return obj; } + private Object[] compatibleConstructorArgs(Object[] fieldValues) { + String[] fieldNames = objectCreator.getConstructorFieldNames(); + Class[] declaringClasses = objectCreator.getConstructorFieldDeclaringClasses(); + Class[] fieldTypes = objectCreator.getConstructorFieldTypes(); + Object[] args = new Object[constructorFieldIndexes.length]; + for (int i = 0; i < constructorFieldIndexes.length; i++) { + int index = constructorFieldIndexes[i]; + if (index >= 0) { + args[i] = fieldValues[index]; + } else { + Class declaringClass = declaringClasses == null ? null : declaringClasses[i]; + args[i] = defaultConstructorValue(fieldNames[i], declaringClass, fieldTypes[i]); + } + } + return args; + } + + private Object defaultConstructorValue( + String fieldName, Class declaringClass, Class fieldType) { + for (DefaultValueUtils.DefaultValueField defaultValueField : defaultValueFields) { + if (defaultValueField.getFieldName().equals(fieldName) + && (declaringClass == null || defaultValueField.getDeclaringClass() == declaringClass)) { + return defaultValueField.getDefaultValue(); + } + } + return AbstractObjectSerializer.defaultConstructorValue(fieldType); + } + @Override public T read(ReadContext readContext) { if (isRecord) { @@ -254,6 +315,10 @@ public T read(ReadContext readContext) { Arrays.fill(recordInfo.getRecordComponents(), null); return t; } + if (objectCreator.hasConstructorFields()) { + Object[] fieldValues = new Object[allFields.length]; + return readConstructorObject(readContext, fieldValues); + } T targetObject = newInstance(); if (readContext.hasPreservedRefId()) { readContext.reference(targetObject); @@ -266,6 +331,96 @@ public T read(ReadContext readContext) { return targetObject; } + private T readConstructorObject(ReadContext readContext, Object[] fieldValues) { + beginConstructorRef(readContext); + try { + boolean[] bufferedNonConstructorFields = new boolean[allFields.length]; + int remainingConstructorFields = countConstructorFields(); + T targetObject = null; + if (remainingConstructorFields == 0) { + targetObject = createConstructorObject(fieldValues); + referenceConstructorRef(readContext, targetObject); + setNonConstructorDefaultValues(targetObject); + } + MemoryBuffer buffer = readContext.getBuffer(); + RefReader refReader = readContext.getRefReader(); + Generics generics = readContext.getGenerics(); + for (int i = 0; i < allFields.length; i++) { + SerializationFieldInfo fieldInfo = allFields[i]; + CompatibleCollectionArrayReader.ReadAction action = + compatibleCollectionArrayReadAction(allCompatibleReadActions, i); + if (constructorFieldMask[i]) { + fieldValues[i] = + ctorFieldValue( + readContext, + readFieldValue(readContext, refReader, generics, fieldInfo, buffer, action), + type); + remainingConstructorFields--; + if (remainingConstructorFields == 0) { + checkNoUnresolvedReadRef(readContext); + targetObject = createConstructorObject(fieldValues); + referenceConstructorRef(readContext, targetObject); + setNonConstructorDefaultValues(targetObject); + setBufferedNonConstructorFields( + targetObject, fieldValues, bufferedNonConstructorFields); + } + } else if (targetObject == null) { + fieldValues[i] = + bufferFieldValue( + readContext, + readFieldValue(readContext, refReader, generics, fieldInfo, buffer, action), + type); + bufferedNonConstructorFields[i] = true; + } else { + readField(readContext, targetObject, refReader, generics, fieldInfo, buffer, action); + } + } + return targetObject; + } finally { + endConstructorRef(readContext); + } + } + + private int countConstructorFields() { + int count = 0; + for (boolean constructorField : constructorFieldMask) { + if (constructorField) { + count++; + } + } + return count; + } + + private T createConstructorObject(Object[] fieldValues) { + return objectCreator.newInstanceWithArguments(compatibleConstructorArgs(fieldValues)); + } + + private void setNonConstructorDefaultValues(T targetObject) { + DefaultValueUtils.setDefaultValues( + targetObject, + defaultValueFields, + objectCreator.getConstructorFieldNames(), + objectCreator.getConstructorFieldDeclaringClasses()); + } + + private void setBufferedNonConstructorFields( + T targetObject, Object[] fieldValues, boolean[] bufferedNonConstructorFields) { + for (int i = 0; i < allFields.length; i++) { + if (bufferedNonConstructorFields[i]) { + setFieldValue( + targetObject, allFields[i], resolveBufferedValue(fieldValues[i], targetObject)); + } + } + } + + private void setFieldValue(T targetObject, SerializationFieldInfo fieldInfo, Object fieldValue) { + if (fieldInfo.fieldAccessor != null) { + fieldInfo.fieldAccessor.putObject(targetObject, fieldValue); + } else if (fieldInfo.fieldConverter != null) { + fieldInfo.fieldConverter.set(targetObject, fieldValue); + } + } + private void readFields(ReadContext readContext, T targetObject) { MemoryBuffer buffer = readContext.getBuffer(); RefReader refReader = readContext.getRefReader(); @@ -275,6 +430,17 @@ private void readFields(ReadContext readContext, T targetObject) { } } + private void readFields(ReadContext readContext, T targetObject, boolean constructorFields) { + MemoryBuffer buffer = readContext.getBuffer(); + RefReader refReader = readContext.getRefReader(); + Generics generics = readContext.getGenerics(); + for (int i = 0; i < allFields.length; i++) { + if (constructorFieldMask[i] == constructorFields) { + readField(readContext, targetObject, refReader, generics, allFields[i], buffer, null); + } + } + } + private void readFields(ReadContext readContext, Object[] fields) { MemoryBuffer buffer = readContext.getBuffer(); int counter = 0; @@ -285,6 +451,17 @@ private void readFields(ReadContext readContext, Object[] fields) { } } + private void readFields(ReadContext readContext, Object[] fields, boolean constructorFields) { + MemoryBuffer buffer = readContext.getBuffer(); + RefReader refReader = readContext.getRefReader(); + Generics generics = readContext.getGenerics(); + for (int i = 0; i < allFields.length; i++) { + if (constructorFieldMask[i] == constructorFields) { + fields[i] = readField(readContext, refReader, generics, allFields[i], buffer, null); + } + } + } + private void compatibleRead( ReadContext readContext, SerializationFieldInfo fieldInfo, Object obj) { MemoryBuffer buffer = readContext.getBuffer(); @@ -309,6 +486,25 @@ private void readFieldsWithCompatibleCollectionArray(ReadContext readContext, T } } + private void readFieldsWithCompatibleCollectionArray( + ReadContext readContext, T targetObject, boolean constructorFields) { + MemoryBuffer buffer = readContext.getBuffer(); + RefReader refReader = readContext.getRefReader(); + Generics generics = readContext.getGenerics(); + for (int i = 0; i < allFields.length; i++) { + if (constructorFieldMask[i] != constructorFields) { + continue; + } + SerializationFieldInfo fieldInfo = allFields[i]; + CompatibleCollectionArrayReader.ReadAction action = + compatibleCollectionArrayReadAction(allCompatibleReadActions, i); + if (Utils.DEBUG_OUTPUT_VERBOSE) { + printFieldDebugInfo(fieldInfo, buffer); + } + readField(readContext, targetObject, refReader, generics, fieldInfo, buffer, action); + } + } + private void readFieldsWithCompatibleCollectionArray(ReadContext readContext, Object[] fields) { MemoryBuffer buffer = readContext.getBuffer(); int counter = 0; @@ -325,6 +521,25 @@ private void readFieldsWithCompatibleCollectionArray(ReadContext readContext, Ob } } + private void readFieldsWithCompatibleCollectionArray( + ReadContext readContext, Object[] fields, boolean constructorFields) { + MemoryBuffer buffer = readContext.getBuffer(); + RefReader refReader = readContext.getRefReader(); + Generics generics = readContext.getGenerics(); + for (int i = 0; i < allFields.length; i++) { + if (constructorFieldMask[i] != constructorFields) { + continue; + } + SerializationFieldInfo fieldInfo = allFields[i]; + CompatibleCollectionArrayReader.ReadAction action = + compatibleCollectionArrayReadAction(allCompatibleReadActions, i); + if (Utils.DEBUG_OUTPUT_ENABLED) { + printFieldDebugInfo(fieldInfo, buffer); + } + fields[i] = readField(readContext, refReader, generics, fieldInfo, buffer, action); + } + } + private void readField( ReadContext readContext, T targetObject, @@ -384,6 +599,23 @@ private Object readField( } } + private Object readFieldValue( + ReadContext readContext, + RefReader refReader, + Generics generics, + SerializationFieldInfo fieldInfo, + MemoryBuffer buffer, + CompatibleCollectionArrayReader.ReadAction action) { + if (fieldInfo.fieldAccessor == null + && fieldInfo.fieldConverter != null + && fieldInfo.codecCategory == FieldGroups.FieldCodecCategory.BUILD_IN + && action == null) { + return AbstractObjectSerializer.readBuildInFieldValue( + readContext, typeResolver, refReader, fieldInfo, buffer); + } + return readField(readContext, refReader, generics, fieldInfo, buffer, action); + } + private void printFieldDebugInfo(SerializationFieldInfo fieldInfo, MemoryBuffer buffer) { LOG.info( "[Java] read field {} of type {}, reader index {}", diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/ExceptionSerializers.java b/java/fory-core/src/main/java/org/apache/fory/serializer/ExceptionSerializers.java index 4c32c84e94..6cdeab3b7a 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/ExceptionSerializers.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/ExceptionSerializers.java @@ -53,9 +53,26 @@ @SuppressWarnings({"rawtypes", "unchecked"}) public final class ExceptionSerializers { private static final Set> THROWABLE_SUPER_CLASSES = ofHashSet(Throwable.class); + private static final ObjectCreator FIELD_ONLY_CREATOR = new FieldOnlyCreator(); private ExceptionSerializers() {} + private static final class FieldOnlyCreator extends ObjectCreator { + private FieldOnlyCreator() { + super(Object.class); + } + + @Override + public Object newInstance() { + throw new UnsupportedOperationException("Throwable layer serializers do not create objects"); + } + + @Override + public Object newInstanceWithArguments(Object... arguments) { + throw new UnsupportedOperationException("Throwable layer serializers do not create objects"); + } + } + public static final class ExceptionSerializer extends Serializer { private final Config config; private final TypeResolver typeResolver; @@ -70,17 +87,18 @@ public ExceptionSerializer(TypeResolver typeResolver, Class type) { this.typeResolver = typeResolver; messageConstructor = getOptionalMessageConstructor(type); objectCreator = - messageConstructor == null && MemoryUtils.JDK_INTERNAL_FIELD_ACCESS + messageConstructor == null && MemoryUtils.JDK_LANG_FIELD_ACCESS ? createThrowableObjectCreator(type) : null; slotsSerializers = buildSlotsSerializers(typeResolver, type); - if (!MemoryUtils.JDK_INTERNAL_FIELD_ACCESS + if (!MemoryUtils.JDK_LANG_FIELD_ACCESS && isJdkThrowable(type) && hasSubclassFields(slotsSerializers)) { throw new ForyException( - "Android doesn't support JDK Throwable type " + "Throwable serialization for JDK type " + type.getName() - + " with subclass fields because JDK private field layouts aren't stable on Android."); + + " with subclass fields requires JDK internal field access. On JDK25+, open " + + "java.base/java.lang to org.apache.fory.core,org.apache.fory.format."); } // Native-image runtime must rebuild slot serializers once so field accessors and // descriptors are created against the runtime heap layout instead of reusing @@ -111,7 +129,7 @@ public void write(WriteContext writeContext, T value) { public T read(ReadContext readContext) { Serializer[] slotsSerializers = getSlotsSerializers(); StackTraceElement[] stackTrace = (StackTraceElement[]) readContext.readRef(); - if (!MemoryUtils.JDK_INTERNAL_FIELD_ACCESS) { + if (!MemoryUtils.JDK_LANG_FIELD_ACCESS) { return readAndroidThrowableWithoutDetailMessageField( readContext, stackTrace, slotsSerializers); } @@ -137,10 +155,11 @@ private T readAndroidThrowableWithoutDetailMessageField( ReadContext readContext, StackTraceElement[] stackTrace, Serializer[] slotsSerializers) { if (messageConstructor == null) { throw new ForyException( - "Android doesn't support deserializing Throwable type " + "Deserializing Throwable type " + type.getName() - + " without a String message constructor because private JDK field access is " - + "unsupported."); + + " without a String message constructor requires JDK internal field access. " + + "On JDK25+, open java.base/java.lang to " + + "org.apache.fory.core,org.apache.fory.format."); } int refId = readContext.lastPreservedRefId(); if (refId >= 0) { @@ -152,9 +171,10 @@ private T readAndroidThrowableWithoutDetailMessageField( skipExtraFields(readContext); if (containsPendingThrowable(cause) || containsPendingThrowable(suppressedExceptions)) { throw new ForyException( - "Android doesn't support deserializing cyclic Throwable references for type " + "Deserializing cyclic Throwable references for type " + type.getName() - + " because private JDK field access is unsupported."); + + " requires JDK internal field access. On JDK25+, open java.base/java.lang " + + "to org.apache.fory.core,org.apache.fory.format."); } T obj = newThrowableWithMessage(detailMessage); readContext.reference(obj); @@ -224,16 +244,17 @@ private void writeNumClassLayers(MemoryBuffer buffer, Serializer[] slotsSerializ public static final class StackTraceElementSerializer extends Serializer { private static final MethodHandles.Lookup LOOKUP = - AndroidSupport.IS_ANDROID ? null : _JDKAccess._trustedLookup(StackTraceElement.class); + AndroidSupport.IS_ANDROID + ? null + : (MemoryUtils.JDK_LANG_FIELD_ACCESS + ? _JDKAccess._trustedLookup(StackTraceElement.class) + : MethodHandles.publicLookup()); private static final MethodHandle CLASS_LOADER_NAME_GETTER = AndroidSupport.IS_ANDROID ? null : getOptionalGetter("getClassLoaderName"); private static final MethodHandle MODULE_NAME_GETTER = getOptionalGetter("getModuleName"); private static final MethodHandle MODULE_VERSION_GETTER = getOptionalGetter("getModuleVersion"); private static final MethodHandle STACK_TRACE_ELEMENT_CTR_V1 = - AndroidSupport.IS_ANDROID - ? null - : ReflectionUtils.getCtrHandle( - StackTraceElement.class, String.class, String.class, String.class, int.class); + getOptionalCtr(String.class, String.class, String.class, int.class); private static final MethodHandle STACK_TRACE_ELEMENT_CTR_V2 = AndroidSupport.IS_ANDROID ? null @@ -350,8 +371,11 @@ private static StackTraceElement newStackTraceElement( fileName, lineNumber); } - return (StackTraceElement) - STACK_TRACE_ELEMENT_CTR_V1.invoke(declaringClass, methodName, fileName, lineNumber); + if (STACK_TRACE_ELEMENT_CTR_V1 != null) { + return (StackTraceElement) + STACK_TRACE_ELEMENT_CTR_V1.invoke(declaringClass, methodName, fileName, lineNumber); + } + return new StackTraceElement(declaringClass, methodName, fileName, lineNumber); } catch (Throwable t) { throw new RuntimeException(t); } @@ -419,7 +443,7 @@ private static Serializer[] buildSlotsSerializers(TypeResolver typeResolver, slotsSerializer = new CompatibleLayerSerializer(typeResolver, type, layerTypeDef, layerMarkerClass); } else { - slotsSerializer = new ObjectSerializer<>(typeResolver, type, false); + slotsSerializer = new ObjectSerializer<>(typeResolver, type, false, fieldOnlyCreator()); } serializers.add(slotsSerializer); type = (Class) type.getSuperclass(); @@ -429,6 +453,10 @@ private static Serializer[] buildSlotsSerializers(TypeResolver typeResolver, return serializers.toArray(new Serializer[0]); } + private static ObjectCreator fieldOnlyCreator() { + return (ObjectCreator) FIELD_ONLY_CREATOR; + } + private static void readAndSetFields( ReadContext readContext, Object target, Serializer[] slotsSerializers, Config config) { readAndCheckNumClassLayers(readContext, target.getClass(), slotsSerializers.length); diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/FinalFieldReplaceResolveSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/FinalFieldReplaceResolveSerializer.java index 92ca339219..40bdb8b535 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/FinalFieldReplaceResolveSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/FinalFieldReplaceResolveSerializer.java @@ -41,13 +41,13 @@ public FinalFieldReplaceResolveSerializer(TypeResolver typeResolver, Class type) @Override protected void writeObject( WriteContext writeContext, Object value, MethodInfoCache jdkMethodInfoCache) { - jdkMethodInfoCache.objectSerializer.write(writeContext, value); + jdkMethodInfoCache.objectSerializer().write(writeContext, value); } @Override protected Object readObject(ReadContext readContext) { MethodInfoCache jdkMethodInfoCache = getMethodInfoCache(type); - Object o = jdkMethodInfoCache.objectSerializer.read(readContext); + Object o = jdkMethodInfoCache.objectSerializer().read(readContext); ReplaceResolveInfo replaceResolveInfo = jdkMethodInfoCache.info; if (replaceResolveInfo.readResolveMethod == null) { return o; diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/JavaSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/JavaSerializer.java index eb0497dc28..5ff4ab37c6 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/JavaSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/JavaSerializer.java @@ -19,6 +19,8 @@ package org.apache.fory.serializer; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; import java.io.Externalizable; import java.io.IOException; import java.io.ObjectInputStream; @@ -30,6 +32,7 @@ import java.nio.ByteBuffer; import org.apache.fory.Fory; import org.apache.fory.collection.ClassValueCache; +import org.apache.fory.context.CopyContext; import org.apache.fory.context.ReadContext; import org.apache.fory.context.WriteContext; import org.apache.fory.io.ClassLoaderObjectInputStream; @@ -51,14 +54,16 @@ *

When a serializer not found and {@link ClassResolver#requireJavaSerialization(Class)} return * true, this serializer will be used. */ -@SuppressWarnings({"rawtypes", "unchecked"}) -public class JavaSerializer extends AbstractObjectSerializer { +@SuppressWarnings({"unchecked"}) +public class JavaSerializer extends Serializer { private static final Logger LOG = LoggerFactory.getLogger(JavaSerializer.class); + private final TypeResolver typeResolver; private final MemoryBufferObjectInput objectInput; private final MemoryBufferObjectOutput objectOutput; public JavaSerializer(TypeResolver typeResolver, Class cls) { - super(typeResolver, cls); + super(typeResolver.getConfig(), (Class) cls); + this.typeResolver = typeResolver; // TODO(chgaokunyang) enable this check when ObjectSerializer is implemented. // Preconditions.checkArgument(ClassResolver.requireJavaSerialization(cls)); if (cls != SerializedLambda.class) { @@ -69,8 +74,8 @@ public JavaSerializer(TypeResolver typeResolver, Class cls) { Serializer.class.getName(), Externalizable.class.getName()); } - objectInput = new MemoryBufferObjectInput(config, null); - objectOutput = new MemoryBufferObjectOutput(config, null); + objectInput = new MemoryBufferObjectInput(typeResolver.getConfig(), null); + objectOutput = new MemoryBufferObjectOutput(typeResolver.getConfig(), null); } @Override @@ -114,6 +119,24 @@ public Object read(ReadContext readContext) { throw new IllegalStateException("unreachable code"); } + @Override + public Object copy(CopyContext copyContext, Object value) { + try { + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + try (ObjectOutputStream output = new ObjectOutputStream(bytes)) { + output.writeObject(value); + } + try (ObjectInputStream input = + new ClassLoaderObjectInputStream( + typeResolver.getClassLoader(), new ByteArrayInputStream(bytes.toByteArray()))) { + return input.readObject(); + } + } catch (IOException | ClassNotFoundException e) { + ExceptionUtils.throwException(e); + throw new IllegalStateException("unreachable code"); + } + } + private static final ClassValueCache writeObjectMethodCache = ClassValueCache.newClassKeyCache(32); diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/JdkProxySerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/JdkProxySerializer.java index aaf1da9156..b2467f44e4 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/JdkProxySerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/JdkProxySerializer.java @@ -117,7 +117,7 @@ public Object copy(CopyContext copyContext, Object value) { Preconditions.checkNotNull(copyHandler); return Proxy.newProxyInstance(typeResolver.getClassLoader(), interfaces, copyHandler); } - if (!MemoryUtils.JDK_INTERNAL_FIELD_ACCESS) { + if (!MemoryUtils.JDK_PROXY_FIELD_ACCESS) { DeferredInvocationHandler deferredHandler = new DeferredInvocationHandler(); Object proxy = Proxy.newProxyInstance(typeResolver.getClassLoader(), interfaces, deferredHandler); @@ -143,7 +143,7 @@ public Object read(ReadContext readContext) { unwrapInvocationHandler((InvocationHandler) readContext.readRef()); return Proxy.newProxyInstance(typeResolver.getClassLoader(), interfaces, invocationHandler); } - if (!MemoryUtils.JDK_INTERNAL_FIELD_ACCESS) { + if (!MemoryUtils.JDK_PROXY_FIELD_ACCESS) { DeferredInvocationHandler deferredHandler = new DeferredInvocationHandler(); Object proxy = Proxy.newProxyInstance(typeResolver.getClassLoader(), interfaces, deferredHandler); diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectSerializer.java index 3830ead2f4..af09950c50 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectSerializer.java @@ -33,6 +33,8 @@ import org.apache.fory.logging.LoggerFactory; import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.meta.TypeDef; +import org.apache.fory.reflect.ObjectCreator; +import org.apache.fory.reflect.ObjectCreators; import org.apache.fory.resolver.TypeResolver; import org.apache.fory.serializer.FieldGroups.SerializationFieldInfo; import org.apache.fory.serializer.struct.Fingerprint; @@ -63,6 +65,8 @@ public final class ObjectSerializer extends AbstractObjectSerializer { private final RecordInfo recordInfo; private final SerializationFieldInfo[] allFields; + private final int[] constructorFieldIndexes; + private final boolean[] constructorFieldMask; private final int classVersionHash; public ObjectSerializer(TypeResolver typeResolver, Class cls) { @@ -70,7 +74,15 @@ public ObjectSerializer(TypeResolver typeResolver, Class cls) { } public ObjectSerializer(TypeResolver typeResolver, Class cls, boolean resolveParent) { - super(typeResolver, cls); + this(typeResolver, cls, resolveParent, ObjectCreators.getObjectCreator(cls)); + } + + public ObjectSerializer( + TypeResolver typeResolver, + Class cls, + boolean resolveParent, + ObjectCreator objectCreator) { + super(typeResolver, cls, objectCreator); // avoid recursive building serializers. // Use `setSerializerIfAbsent` to avoid overwriting existing serializer for class when used // as data serializer. @@ -126,6 +138,13 @@ public ObjectSerializer(TypeResolver typeResolver, Class cls, boolean resolve } FieldGroups fieldGroups = FieldGroups.buildFieldInfos(typeResolver, grouper); allFields = fieldGroups.allFields; + if (!isRecord && objectCreator.hasConstructorFields()) { + constructorFieldIndexes = buildConstructorFieldIndexes(allFields); + constructorFieldMask = buildConstructorFieldMask(allFields.length, constructorFieldIndexes); + } else { + constructorFieldIndexes = null; + constructorFieldMask = null; + } } @Override @@ -137,11 +156,29 @@ public void write(WriteContext writeContext, T value) { // Protocol order: primitive, nullable primitive, then all non-primitives by field identifier. RefWriter refWriter = writeContext.getRefWriter(); Generics generics = writeContext.getGenerics(); + writeFields(writeContext, value, refWriter, generics); + } + + private void writeFields( + WriteContext writeContext, T value, RefWriter refWriter, Generics generics) { for (SerializationFieldInfo fieldInfo : allFields) { writeFieldByCodecCategory(writeContext, value, refWriter, generics, fieldInfo); } } + private void writeFields( + WriteContext writeContext, + T value, + RefWriter refWriter, + Generics generics, + boolean constructorFields) { + for (int i = 0; i < allFields.length; i++) { + if (constructorFieldMask[i] == constructorFields) { + writeFieldByCodecCategory(writeContext, value, refWriter, generics, allFields[i]); + } + } + } + private void printWriteFieldDebugInfo(SerializationFieldInfo fieldInfo, MemoryBuffer buffer) { LOG.info( "[Java] write field {} of type {}, writer index {}", @@ -202,11 +239,89 @@ public T read(ReadContext readContext) { Arrays.fill(recordInfo.getRecordComponents(), null); return obj; } + if (objectCreator.hasConstructorFields()) { + return readConstructorObject(readContext); + } T obj = newBean(); readContext.reference(obj); return readAndSetFields(readContext, obj); } + private T readConstructorObject(ReadContext readContext) { + beginConstructorRef(readContext); + try { + MemoryBuffer buffer = readContext.getBuffer(); + if (typeResolver.checkClassVersion()) { + int hash = buffer.readInt32(); + checkClassVersion(type, hash, classVersionHash); + } + Object[] fieldValues = new Object[allFields.length]; + boolean[] bufferedNonConstructorFields = new boolean[allFields.length]; + int remainingConstructorFields = countConstructorFields(); + T obj = null; + if (remainingConstructorFields == 0) { + obj = createConstructorObject(fieldValues); + referenceConstructorRef(readContext, obj); + } + RefReader refReader = readContext.getRefReader(); + Generics generics = readContext.getGenerics(); + for (int i = 0; i < allFields.length; i++) { + SerializationFieldInfo fieldInfo = allFields[i]; + if (constructorFieldMask[i]) { + fieldValues[i] = + ctorFieldValue( + readContext, + readFieldByCodecCategory(readContext, refReader, generics, fieldInfo, buffer), + type); + remainingConstructorFields--; + if (remainingConstructorFields == 0) { + checkNoUnresolvedReadRef(readContext); + obj = createConstructorObject(fieldValues); + referenceConstructorRef(readContext, obj); + setBufferedNonConstructorFields(obj, fieldValues, bufferedNonConstructorFields); + } + } else if (obj == null) { + fieldValues[i] = + bufferFieldValue( + readContext, + readFieldByCodecCategory(readContext, refReader, generics, fieldInfo, buffer), + type); + bufferedNonConstructorFields[i] = true; + } else { + readAndSetFieldByCodecCategory(readContext, refReader, generics, fieldInfo, buffer, obj); + } + } + return obj; + } finally { + endConstructorRef(readContext); + } + } + + private int countConstructorFields() { + int count = 0; + for (boolean constructorField : constructorFieldMask) { + if (constructorField) { + count++; + } + } + return count; + } + + private T createConstructorObject(Object[] fieldValues) { + return objectCreator.newInstanceWithArguments( + constructorArgs( + fieldValues, constructorFieldIndexes, objectCreator.getConstructorFieldTypes())); + } + + private void setBufferedNonConstructorFields( + T obj, Object[] fieldValues, boolean[] bufferedNonConstructorFields) { + for (int i = 0; i < allFields.length; i++) { + if (bufferedNonConstructorFields[i]) { + allFields[i].fieldAccessor.putObject(obj, resolveBufferedValue(fieldValues[i], obj)); + } + } + } + public Object[] readFields(ReadContext readContext) { MemoryBuffer buffer = readContext.getBuffer(); RefReader refReader = readContext.getRefReader(); @@ -224,6 +339,19 @@ public Object[] readFields(ReadContext readContext) { return fieldValues; } + private void readFields( + ReadContext readContext, Object[] fieldValues, boolean constructorFields) { + MemoryBuffer buffer = readContext.getBuffer(); + RefReader refReader = readContext.getRefReader(); + Generics generics = readContext.getGenerics(); + for (int i = 0; i < allFields.length; i++) { + if (constructorFieldMask[i] == constructorFields) { + fieldValues[i] = + readFieldByCodecCategory(readContext, refReader, generics, allFields[i], buffer); + } + } + } + public T readAndSetFields(ReadContext readContext, T obj) { MemoryBuffer buffer = readContext.getBuffer(); RefReader refReader = readContext.getRefReader(); @@ -238,6 +366,17 @@ public T readAndSetFields(ReadContext readContext, T obj) { return obj; } + private void readAndSetFields(ReadContext readContext, T obj, boolean constructorFields) { + MemoryBuffer buffer = readContext.getBuffer(); + RefReader refReader = readContext.getRefReader(); + Generics generics = readContext.getGenerics(); + for (int i = 0; i < allFields.length; i++) { + if (constructorFieldMask[i] == constructorFields) { + readAndSetFieldByCodecCategory(readContext, refReader, generics, allFields[i], buffer, obj); + } + } + } + private Object readFieldByCodecCategory( ReadContext readContext, RefReader refReader, diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectStreamSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectStreamSerializer.java index a5f1fe852a..09b34f4318 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectStreamSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectStreamSerializer.java @@ -39,6 +39,7 @@ import java.util.Collection; import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.TreeMap; import java.util.function.BiConsumer; import java.util.function.Consumer; @@ -49,6 +50,7 @@ import org.apache.fory.collection.ObjectArray; import org.apache.fory.collection.ObjectIntMap; import org.apache.fory.config.Int64Encoding; +import org.apache.fory.context.CopyContext; import org.apache.fory.context.MetaReadContext; import org.apache.fory.context.ReadContext; import org.apache.fory.context.WriteContext; @@ -56,20 +58,22 @@ import org.apache.fory.logging.Logger; import org.apache.fory.logging.LoggerFactory; import org.apache.fory.memory.MemoryBuffer; +import org.apache.fory.memory.MemoryUtils; import org.apache.fory.meta.FieldInfo; import org.apache.fory.meta.FieldTypes; import org.apache.fory.meta.NativeTypeDefEncoder; import org.apache.fory.meta.TypeDef; import org.apache.fory.platform.AndroidSupport; import org.apache.fory.platform.GraalvmSupport; +import org.apache.fory.platform.JdkVersion; import org.apache.fory.platform.internal._JDKAccess; import org.apache.fory.reflect.ObjectCreator; import org.apache.fory.reflect.ObjectCreators; -import org.apache.fory.reflect.ReflectionUtils; import org.apache.fory.resolver.ClassResolver; import org.apache.fory.resolver.TypeInfo; import org.apache.fory.resolver.TypeResolver; import org.apache.fory.serializer.PrimitiveSerializers.LongSerializer; +import org.apache.fory.serializer.collection.ChildContainerSerializers; import org.apache.fory.type.Descriptor; import org.apache.fory.type.TypeUtils; import org.apache.fory.type.Types; @@ -94,6 +98,7 @@ public class ObjectStreamSerializer extends AbstractObjectSerializer { private static final int MAX_CACHED_TYPE_DEFS = 8192; private final SlotInfo[] slotsInfos; + private final Serializer fallbackSerializer; // Instance-level cache: TypeDef ID -> TypeInfo (shared across all slots). private final LongMap typeDefIdToTypeInfo = new LongMap<>(4, 0.4f); @@ -184,11 +189,17 @@ private static ObjectStreamClass safeObjectStreamClassLookup(Class type) { } public ObjectStreamSerializer(TypeResolver typeResolver, Class type) { - super(typeResolver, type, createObjectCreatorForGraalVM(type)); + super(typeResolver, type, createObjectStreamCreator(type)); if (!Serializable.class.isAssignableFrom(type)) { throw new IllegalArgumentException( String.format("Class %s should implement %s.", type, Serializable.class)); } + Serializer fallbackSerializer = fallbackSerializer(typeResolver, type); + this.fallbackSerializer = fallbackSerializer; + if (fallbackSerializer != null) { + slotsInfos = new SlotInfo[0]; + return; + } if (!Throwable.class.isAssignableFrom(type)) { LOG.warn( "{} customized jdk serialization, which is inefficient. " @@ -213,13 +224,35 @@ public ObjectStreamSerializer(TypeResolver typeResolver, Class type) { slotsInfos = slotsInfoList.toArray(new SlotInfo[0]); } - /** - * Creates an appropriate ObjectCreator for GraalVM native image environment. In GraalVM, we - * prefer UnsafeObjectCreator to avoid serialization constructor issues. - */ - private static ObjectCreator createObjectCreatorForGraalVM(Class type) { - if (GraalvmSupport.IN_GRAALVM_NATIVE_IMAGE) { - // In GraalVM native image, use Unsafe to avoid serialization constructor issues + private static Serializer fallbackSerializer(TypeResolver typeResolver, Class type) { + if (JdkVersion.MAJOR_VERSION < 25) { + return null; + } + Class childSerializerClass = null; + if (Collection.class.isAssignableFrom(type)) { + childSerializerClass = ChildContainerSerializers.getCollectionSerializerClass(type); + } else if (Map.class.isAssignableFrom(type)) { + childSerializerClass = ChildContainerSerializers.getMapSerializerClass(type); + } + if (childSerializerClass != null) { + return Serializers.newSerializer(typeResolver, type, childSerializerClass); + } + if (type.getName().startsWith("java.")) { + return new JavaSerializer(typeResolver, type); + } + return null; + } + + /** Creates an ObjectCreator for Java ObjectStream-compatible reconstruction. */ + private static ObjectCreator createObjectStreamCreator(Class type) { + if (JdkVersion.MAJOR_VERSION >= 25) { + if (hasJdk25Fallback(type)) { + return new FallbackOnlyObjectCreator<>(type); + } + // ObjectStreamSerializer must preserve Java serialization construction semantics. On JDK25+ + // this path cannot fall back to Unsafe, including inside GraalVM native images. + return new ObjectCreators.ParentNoArgCtrObjectCreator<>(type); + } else if (GraalvmSupport.IN_GRAALVM_NATIVE_IMAGE) { return new ObjectCreators.UnsafeObjectCreator<>(type); } else { // In regular JVM, use the standard object creator @@ -227,8 +260,44 @@ private static ObjectCreator createObjectCreatorForGraalVM(Class type) } } + private static boolean hasJdk25Fallback(Class type) { + if (JdkVersion.MAJOR_VERSION < 25) { + return false; + } + if (type.getName().startsWith("java.")) { + return true; + } + if (Collection.class.isAssignableFrom(type)) { + return ChildContainerSerializers.getCollectionSerializerClass(type) != null; + } + if (Map.class.isAssignableFrom(type)) { + return ChildContainerSerializers.getMapSerializerClass(type) != null; + } + return false; + } + + private static final class FallbackOnlyObjectCreator extends ObjectCreator { + private FallbackOnlyObjectCreator(Class type) { + super(type); + } + + @Override + public T newInstance() { + throw new ForyException("ObjectStreamSerializer fallback owns construction for " + type); + } + + @Override + public T newInstanceWithArguments(Object... arguments) { + throw new ForyException("ObjectStreamSerializer fallback owns construction for " + type); + } + } + @Override public void write(WriteContext writeContext, Object value) { + if (fallbackSerializer != null) { + fallbackSerializer.write(writeContext, value); + return; + } MemoryBuffer buffer = writeContext.getBuffer(); buffer.writeInt16((short) slotsInfos.length); try { @@ -281,6 +350,9 @@ public void write(WriteContext writeContext, Object value) { @Override public Object read(ReadContext readContext) { + if (fallbackSerializer != null) { + return fallbackSerializer.read(readContext); + } MemoryBuffer buffer = readContext.getBuffer(); Object obj = objectCreator.newInstance(); readContext.reference(obj); @@ -399,6 +471,14 @@ public Object read(ReadContext readContext) { return obj; } + @Override + public Object copy(CopyContext copyContext, Object value) { + if (fallbackSerializer != null) { + return fallbackSerializer.copy(copyContext, value); + } + return super.copy(copyContext, value); + } + /** * Skip data for a layer that exists in sender but not in receiver. This is needed for schema * evolution when sender's class hierarchy has layers that receiver doesn't have. @@ -610,17 +690,14 @@ private StreamTypeInfo(Class type) { Method writeMethod = null; Method readMethod = null; Method noDataMethod = null; - if (AndroidSupport.IS_ANDROID) { + if (AndroidSupport.IS_ANDROID || !MemoryUtils.JDK_OBJECT_STREAM_FIELD_ACCESS) { writeMethod = JavaSerializer.getWriteObjectMethod(type, false); readMethod = JavaSerializer.getReadRefMethod(type, false); noDataMethod = JavaSerializer.getReadRefNoData(type, false); } else if (objectStreamClass != null) { - writeMethod = - (Method) ReflectionUtils.getObjectFieldValue(objectStreamClass, "writeObjectMethod"); - readMethod = - (Method) ReflectionUtils.getObjectFieldValue(objectStreamClass, "readObjectMethod"); - noDataMethod = - (Method) ReflectionUtils.getObjectFieldValue(objectStreamClass, "readObjectNoData"); + writeMethod = _JDKAccess.getObjectStreamClassWriteObjectMethod(objectStreamClass); + readMethod = _JDKAccess.getObjectStreamClassReadObjectMethod(objectStreamClass); + noDataMethod = _JDKAccess.getObjectStreamClassReadObjectNoDataMethod(objectStreamClass); } this.writeObjectMethod = writeMethod; this.readObjectMethod = readMethod; diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/ReplaceResolveSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/ReplaceResolveSerializer.java index 03d6492efa..598977c545 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/ReplaceResolveSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/ReplaceResolveSerializer.java @@ -36,9 +36,11 @@ import org.apache.fory.logging.Logger; import org.apache.fory.logging.LoggerFactory; import org.apache.fory.memory.MemoryBuffer; +import org.apache.fory.memory.MemoryUtils; import org.apache.fory.platform.AndroidSupport; +import org.apache.fory.platform.JdkVersion; import org.apache.fory.platform.internal._JDKAccess; -import org.apache.fory.reflect.ReflectionUtils; +import org.apache.fory.reflect.ObjectCreators; import org.apache.fory.resolver.ClassResolver; import org.apache.fory.resolver.TypeInfo; import org.apache.fory.resolver.TypeResolver; @@ -75,15 +77,13 @@ private ReplaceResolveInfo(Class cls) { Method writeReplaceMethod, readResolveMethod; // In JDK17, set private jdk method accessible will fail by default, use ObjectStreamClass // instead, since it set accessible. - if (AndroidSupport.IS_ANDROID) { + if (AndroidSupport.IS_ANDROID || !MemoryUtils.JDK_OBJECT_STREAM_FIELD_ACCESS) { writeReplaceMethod = JavaSerializer.getWriteReplaceMethod(cls); readResolveMethod = JavaSerializer.getReadResolveMethod(cls); } else if (Serializable.class.isAssignableFrom(cls)) { ObjectStreamClass objectStreamClass = ObjectStreamClass.lookup(cls); - writeReplaceMethod = - (Method) ReflectionUtils.getObjectFieldValue(objectStreamClass, "writeReplaceMethod"); - readResolveMethod = - (Method) ReflectionUtils.getObjectFieldValue(objectStreamClass, "readResolveMethod"); + writeReplaceMethod = _JDKAccess.getObjectStreamClassWriteReplaceMethod(objectStreamClass); + readResolveMethod = _JDKAccess.getObjectStreamClassReadResolveMethod(objectStreamClass); } else { // FIXME class with `writeReplace` method defined should be Serializable, // but hessian ignores this check and many existing system are using hessian, @@ -113,7 +113,7 @@ private ReplaceResolveInfo(Class cls) { : (readResolveMethod != null ? readResolveMethod.getDeclaringClass() : null); Function writeReplaceFunc = null, readResolveFunc = null; if (declaringClass != null) { - if (AndroidSupport.IS_ANDROID) { + if (AndroidSupport.IS_ANDROID || !MemoryUtils.JDK_OBJECT_STREAM_FIELD_ACCESS) { makeAccessible(writeReplaceMethod); makeAccessible(readResolveMethod); } else { @@ -184,38 +184,88 @@ Object readResolve(Object o) { protected static class MethodInfoCache { protected final ReplaceResolveInfo info; + private final TypeResolver typeResolver; + private final Class cls; + private Class serializerClass; - protected Serializer objectSerializer; + protected volatile Serializer objectSerializer; - public MethodInfoCache(ReplaceResolveInfo info) { + public MethodInfoCache(ReplaceResolveInfo info, TypeResolver typeResolver, Class cls) { this.info = info; + this.typeResolver = typeResolver; + this.cls = cls; + } + + public void setSerializerClass(Class serializerClass) { + this.serializerClass = serializerClass; } public void setObjectSerializer(Serializer objectSerializer) { this.objectSerializer = objectSerializer; } + + public Serializer objectSerializer() { + Serializer serializer = objectSerializer; + if (serializer == null) { + synchronized (this) { + serializer = objectSerializer; + if (serializer == null) { + Class sc = serializerClass; + if (sc == null) { + sc = serializerClass = dataSerializerClass(typeResolver, cls, this); + serializer = objectSerializer; + if (serializer != null) { + return serializer; + } + } + serializer = createDataSerializer(typeResolver, cls, sc); + objectSerializer = serializer; + } + } + } + return serializer; + } } static MethodInfoCache newJDKMethodInfoCache(TypeResolver typeResolver, Class cls) { ReplaceResolveInfo replaceResolveInfo = REPLACE_RESOLVE_INFO_CACHE.get(cls, () -> new ReplaceResolveInfo(cls)); - MethodInfoCache methodInfoCache = new MethodInfoCache(replaceResolveInfo); + MethodInfoCache methodInfoCache = new MethodInfoCache(replaceResolveInfo, typeResolver, cls); + ClassResolver classResolver = (ClassResolver) typeResolver; + Serializer registeredSerializer = classResolver.getSerializer(cls, false); + if (registeredSerializer != null + && !(registeredSerializer instanceof ReplaceResolveSerializer)) { + methodInfoCache.setObjectSerializer(registeredSerializer); + return methodInfoCache; + } + methodInfoCache.setSerializerClass(null); + return methodInfoCache; + } + + private static Class dataSerializerClass( + TypeResolver typeResolver, Class cls, MethodInfoCache methodInfoCache) { ClassResolver classResolver = (ClassResolver) typeResolver; Class serializerClass; if (Externalizable.class.isAssignableFrom(cls)) { serializerClass = ExternalizableSerializer.class; } else if (JavaSerializer.getReadRefMethod(cls, true) == null && JavaSerializer.getWriteObjectMethod(cls, true) == null) { - serializerClass = - classResolver.getObjectSerializerClass( - cls, - sc -> - methodInfoCache.setObjectSerializer(createDataSerializer(typeResolver, cls, sc))); + if (JdkVersion.MAJOR_VERSION >= 25 + && Serializable.class.isAssignableFrom(cls) + && !ObjectCreators.supportsJdk25Creation(cls)) { + serializerClass = typeResolver.getDefaultJDKStreamSerializerType(); + } else { + serializerClass = + classResolver.getObjectSerializerClass( + cls, + sc -> + methodInfoCache.setObjectSerializer( + createDataSerializer(typeResolver, cls, sc))); + } } else { serializerClass = typeResolver.getDefaultJDKStreamSerializerType(); } - methodInfoCache.setObjectSerializer(createDataSerializer(typeResolver, cls, serializerClass)); - return methodInfoCache; + return serializerClass; } /** @@ -320,7 +370,7 @@ public void write(WriteContext writeContext, Object value) { protected void writeObject( WriteContext writeContext, Object value, MethodInfoCache jdkMethodInfoCache) { classResolver.writeClassInternal(writeContext, writeTypeInfo); - jdkMethodInfoCache.objectSerializer.write(writeContext, value); + jdkMethodInfoCache.objectSerializer().write(writeContext, value); } @Override @@ -360,7 +410,7 @@ public Object read(ReadContext readContext) { protected Object readObject(ReadContext readContext) { Class cls = classResolver.readClassInternal(readContext); MethodInfoCache jdkMethodInfoCache = getMethodInfoCache(cls); - Object o = jdkMethodInfoCache.objectSerializer.read(readContext); + Object o = jdkMethodInfoCache.objectSerializer().read(readContext); ReplaceResolveInfo replaceResolveInfo = jdkMethodInfoCache.info; if (replaceResolveInfo.readResolveMethod == null) { return o; @@ -372,7 +422,7 @@ protected Object readObject(ReadContext readContext) { public Object copy(CopyContext copyContext, Object originObj) { ReplaceResolveInfo replaceResolveInfo = jdkMethodInfoWriteCache.info; if (replaceResolveInfo.writeReplaceMethod == null) { - return jdkMethodInfoWriteCache.objectSerializer.copy(copyContext, originObj); + return jdkMethodInfoWriteCache.objectSerializer().copy(copyContext, originObj); } Object newObj = originObj; newObj = replaceResolveInfo.writeReplace(newObj); @@ -380,7 +430,7 @@ public Object copy(CopyContext copyContext, Object originObj) { copyContext.reference(originObj, newObj); } MethodInfoCache jdkMethodInfoCache = getMethodInfoCache(newObj.getClass()); - newObj = jdkMethodInfoCache.objectSerializer.copy(copyContext, newObj); + newObj = jdkMethodInfoCache.objectSerializer().copy(copyContext, newObj); replaceResolveInfo = jdkMethodInfoCache.info; if (replaceResolveInfo.readResolveMethod != null) { newObj = replaceResolveInfo.readResolve(newObj); diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/Serializers.java b/java/fory-core/src/main/java/org/apache/fory/serializer/Serializers.java index 0f534f79bd..2cca216fba 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/Serializers.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/Serializers.java @@ -31,6 +31,7 @@ import java.net.URI; import java.nio.charset.Charset; import java.util.Currency; +import java.util.StringTokenizer; import java.util.UUID; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; @@ -48,10 +49,12 @@ import org.apache.fory.context.ReadContext; import org.apache.fory.context.WriteContext; import org.apache.fory.memory.MemoryBuffer; +import org.apache.fory.memory.MemoryUtils; import org.apache.fory.meta.TypeDef; import org.apache.fory.platform.AndroidSupport; import org.apache.fory.platform.GraalvmSupport; import org.apache.fory.platform.internal._JDKAccess; +import org.apache.fory.reflect.FieldAccessor; import org.apache.fory.reflect.ReflectionUtils; import org.apache.fory.resolver.ClassResolver; import org.apache.fory.resolver.TypeInfo; @@ -501,6 +504,88 @@ public StringBuffer read(ReadContext readContext) { } } + public static final class StringTokenizerSerializer extends Serializer + implements Shareable { + public StringTokenizerSerializer(Config config) { + super(config, StringTokenizer.class); + } + + @Override + public void write(WriteContext writeContext, StringTokenizer value) { + checkStringTokenizerAccess(); + MemoryBuffer buffer = writeContext.getBuffer(); + writeContext.writeRef(Accessors.STR.getObject(value)); + writeContext.writeRef(Accessors.DELIMITERS.getObject(value)); + buffer.writeBoolean(Accessors.RET_DELIMS.getBoolean(value)); + buffer.writeVarInt32(Accessors.CURRENT_POSITION.getInt(value)); + buffer.writeVarInt32(Accessors.NEW_POSITION.getInt(value)); + buffer.writeBoolean(Accessors.DELIMS_CHANGED.getBoolean(value)); + } + + @Override + public StringTokenizer read(ReadContext readContext) { + checkStringTokenizerAccess(); + String str = (String) readContext.readRef(); + String delimiters = (String) readContext.readRef(); + boolean retDelims = readContext.getBuffer().readBoolean(); + StringTokenizer tokenizer = new StringTokenizer(str, delimiters, retDelims); + restoreState(readContext.getBuffer(), tokenizer); + return tokenizer; + } + + @Override + public StringTokenizer copy(CopyContext copyContext, StringTokenizer value) { + checkStringTokenizerAccess(); + StringTokenizer tokenizer = + new StringTokenizer( + (String) Accessors.STR.getObject(value), + (String) Accessors.DELIMITERS.getObject(value), + Accessors.RET_DELIMS.getBoolean(value)); + Accessors.CURRENT_POSITION.putInt(tokenizer, Accessors.CURRENT_POSITION.getInt(value)); + Accessors.NEW_POSITION.putInt(tokenizer, Accessors.NEW_POSITION.getInt(value)); + Accessors.DELIMS_CHANGED.putBoolean(tokenizer, Accessors.DELIMS_CHANGED.getBoolean(value)); + return tokenizer; + } + + private static void restoreState(MemoryBuffer buffer, StringTokenizer tokenizer) { + Accessors.CURRENT_POSITION.putInt(tokenizer, buffer.readVarInt32()); + Accessors.NEW_POSITION.putInt(tokenizer, buffer.readVarInt32()); + Accessors.DELIMS_CHANGED.putBoolean(tokenizer, buffer.readBoolean()); + } + + private static void checkStringTokenizerAccess() { + if (!MemoryUtils.JDK_COLLECTION_FIELD_ACCESS) { + throw stringTokenizerAccessError(); + } + } + + private static UnsupportedOperationException stringTokenizerAccessError() { + return new UnsupportedOperationException( + "StringTokenizer serialization requires JDK internal field access. On JDK25+, open " + + "java.base/java.util to org.apache.fory.core,org.apache.fory.format."); + } + + private static final class Accessors { + private static final FieldAccessor CURRENT_POSITION = + FieldAccessor.createAccessor( + ReflectionUtils.getField(StringTokenizer.class, "currentPosition")); + private static final FieldAccessor NEW_POSITION = + FieldAccessor.createAccessor( + ReflectionUtils.getField(StringTokenizer.class, "newPosition")); + private static final FieldAccessor STR = + FieldAccessor.createAccessor(ReflectionUtils.getField(StringTokenizer.class, "str")); + private static final FieldAccessor DELIMITERS = + FieldAccessor.createAccessor( + ReflectionUtils.getField(StringTokenizer.class, "delimiters")); + private static final FieldAccessor RET_DELIMS = + FieldAccessor.createAccessor( + ReflectionUtils.getField(StringTokenizer.class, "retDelims")); + private static final FieldAccessor DELIMS_CHANGED = + FieldAccessor.createAccessor( + ReflectionUtils.getField(StringTokenizer.class, "delimsChanged")); + } + } + public static final class AtomicBooleanSerializer extends Serializer implements Shareable { @@ -731,6 +816,10 @@ public static void registerDefaultSerializers(TypeResolver resolver) { resolver.registerInternalSerializer(Class.class, new ClassSerializer(config)); resolver.registerInternalSerializer(StringBuilder.class, new StringBuilderSerializer(config)); resolver.registerInternalSerializer(StringBuffer.class, new StringBufferSerializer(config)); + // Keep this internal type id reserved even when JDK collection internals are not open; + // otherwise payloads written with access enabled decode later collection ids incorrectly. + resolver.registerInternalSerializer( + StringTokenizer.class, new StringTokenizerSerializer(config)); resolver.registerInternalSerializer(BigInteger.class, new BigIntegerSerializer(config)); resolver.registerInternalSerializer(BigDecimal.class, new DecimalSerializer(config)); resolver.registerInternalSerializer(AtomicBoolean.class, new AtomicBooleanSerializer(config)); diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/StringSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/StringSerializer.java index f2fcac8805..8e9fa43965 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/StringSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/StringSerializer.java @@ -23,15 +23,8 @@ import static org.apache.fory.util.StringUtils.MULTI_CHARS_NON_ASCII_MASK; import static org.apache.fory.util.StringUtils.MULTI_CHARS_NON_LATIN_MASK; -import java.lang.invoke.CallSite; -import java.lang.invoke.LambdaMetafactory; -import java.lang.invoke.MethodHandle; -import java.lang.invoke.MethodHandles; -import java.lang.invoke.MethodType; import java.nio.charset.StandardCharsets; import java.util.Arrays; -import java.util.function.BiFunction; -import java.util.function.Function; import org.apache.fory.annotation.CodegenInvoke; import org.apache.fory.codegen.Expression; import org.apache.fory.codegen.Expression.Invoke; @@ -43,7 +36,6 @@ import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.memory.NativeByteOrder; import org.apache.fory.platform.AndroidSupport; -import org.apache.fory.platform.JdkVersion; import org.apache.fory.platform.UnsafeOps; import org.apache.fory.platform.internal._JDKAccess; import org.apache.fory.util.MathUtils; @@ -52,7 +44,7 @@ import org.apache.fory.util.StringUtils; /** - * String serializer based on {@link sun.misc.Unsafe} and {@link MethodHandle} for speed. + * String serializer based on JDK-internal string access and byte-array accessors for speed. * *

Note that string operations is very common in serialization, and jvm inline and branch * elimination is not reliable even in c2 compiler, so we try to inline and avoid checks as we can @@ -64,28 +56,35 @@ public final class StringSerializer extends ImmutableSerializer { private static final boolean STRING_VALUE_FIELD_IS_BYTES; private static final byte LATIN1 = 0; - private static final Byte LATIN1_BOXED = LATIN1; private static final byte UTF16 = 1; - private static final Byte UTF16_BOXED = UTF16; private static final byte UTF8 = 2; private static final int DEFAULT_BUFFER_SIZE = 1024; + private static final long STRING_VALUE_FIELD_OFFSET; private static final boolean STRING_HAS_COUNT_OFFSET; + private static final long STRING_COUNT_FIELD_OFFSET; + private static final long STRING_OFFSET_FIELD_OFFSET; static { if (!jdkInternalFieldAccess()) { STRING_VALUE_FIELD_IS_CHARS = false; STRING_VALUE_FIELD_IS_BYTES = false; + STRING_VALUE_FIELD_OFFSET = -1; STRING_HAS_COUNT_OFFSET = false; + STRING_COUNT_FIELD_OFFSET = -1; + STRING_OFFSET_FIELD_OFFSET = -1; } else { STRING_VALUE_FIELD_IS_CHARS = _JDKAccess.STRING_VALUE_FIELD_IS_CHARS; STRING_VALUE_FIELD_IS_BYTES = _JDKAccess.STRING_VALUE_FIELD_IS_BYTES; + STRING_VALUE_FIELD_OFFSET = _JDKAccess.STRING_VALUE_FIELD_OFFSET; STRING_HAS_COUNT_OFFSET = _JDKAccess.STRING_HAS_COUNT_OFFSET; + STRING_COUNT_FIELD_OFFSET = _JDKAccess.STRING_COUNT_FIELD_OFFSET; + STRING_OFFSET_FIELD_OFFSET = _JDKAccess.STRING_OFFSET_FIELD_OFFSET; } } private static boolean jdkInternalFieldAccess() { - return !AndroidSupport.IS_ANDROID && _JDKAccess.JDK_INTERNAL_FIELD_ACCESS; + return !AndroidSupport.IS_ANDROID && _JDKAccess.JDK_STRING_FIELD_ACCESS; } private final boolean compressString; @@ -427,19 +426,19 @@ private static boolean isLatin(char[] chars) { } private static Object getStringValue(String value) { - return _JDKAccess.getStringValue(value); + return UnsafeOps.getObject(value, STRING_VALUE_FIELD_OFFSET); } private static byte getStringCoder(String value) { - return _JDKAccess.getStringCoder(value); + return UnsafeOps.getByte(value, _JDKAccess.STRING_CODER_FIELD_OFFSET); } private static int getStringOffset(String value) { - return _JDKAccess.getStringOffset(value); + return UnsafeOps.getInt(value, STRING_OFFSET_FIELD_OFFSET); } private static int getStringCount(String value) { - return _JDKAccess.getStringCount(value); + return UnsafeOps.getInt(value, STRING_COUNT_FIELD_OFFSET); } @CodegenInvoke @@ -885,24 +884,11 @@ private void writeBytesUTF8PerfOptimized(MemoryBuffer buffer, byte[] bytes) { } } - private static final MethodHandles.Lookup STRING_LOOK_UP = - jdkInternalFieldAccess() ? _JDKAccess._trustedLookup(String.class) : null; - private static final BiFunction CHARS_STRING_ZERO_COPY_CTR = - jdkInternalFieldAccess() ? getCharsStringZeroCopyCtr() : null; - private static final BiFunction BYTES_STRING_ZERO_COPY_CTR = - jdkInternalFieldAccess() ? getBytesStringZeroCopyCtr() : null; - private static final Function LATIN_BYTES_STRING_ZERO_COPY_CTR = - jdkInternalFieldAccess() ? getLatinBytesStringZeroCopyCtr() : null; - public static String newCharsStringZeroCopy(char[] data) { if (!jdkInternalFieldAccess()) { return newCharsStringSlow(data); } - if (!STRING_VALUE_FIELD_IS_CHARS) { - throw new IllegalStateException("String value isn't char[], current java isn't supported"); - } - // 25% faster than unsafe put field, only 10% slower than `new String(str)` - return CHARS_STRING_ZERO_COPY_CTR.apply(data, Boolean.TRUE); + return _JDKAccess.newCharsStringZeroCopy(data); } private static String newCharsStringSlow(char[] data) { @@ -915,27 +901,7 @@ public static String newBytesStringZeroCopy(byte coder, byte[] data) { if (!jdkInternalFieldAccess()) { return newBytesStringSlow(coder, data); } - if (coder == LATIN1) { - // 700% faster than unsafe put field in java11, only 10% slower than `new String(str)` for - // string length 230. - // 50% faster than unsafe put field in java11 for string length 10. - if (LATIN_BYTES_STRING_ZERO_COPY_CTR != null) { - return LATIN_BYTES_STRING_ZERO_COPY_CTR.apply(data); - } else { - // JDK17 removed newStringLatin1 - return BYTES_STRING_ZERO_COPY_CTR.apply(data, LATIN1_BOXED); - } - } else if (coder == UTF16) { - // avoid byte box cost. - return BYTES_STRING_ZERO_COPY_CTR.apply(data, UTF16_BOXED); - } else { - // 700% faster than unsafe put field in java11, only 10% slower than `new String(str)` for - // string length 230. - // 50% faster than unsafe put field in java11 for string length 10. - // `invokeExact` must pass exact params with exact types: - // `(Object) data, coder` will throw WrongMethodTypeException - return BYTES_STRING_ZERO_COPY_CTR.apply(data, coder); - } + return _JDKAccess.newBytesStringZeroCopy(coder, data); } private static String newBytesStringSlow(byte coder, byte[] data) { @@ -952,95 +918,6 @@ private static String newBytesStringSlow(byte coder, byte[] data) { } } - private static BiFunction getCharsStringZeroCopyCtr() { - if (!STRING_VALUE_FIELD_IS_CHARS) { - return null; - } - MethodHandle handle = getJavaStringZeroCopyCtrHandle(); - if (handle == null) { - return null; - } - try { - // Faster than handle.invokeExact(data, boolean) - CallSite callSite = - LambdaMetafactory.metafactory( - STRING_LOOK_UP, - "apply", - MethodType.methodType(BiFunction.class), - handle.type().generic(), - handle, - handle.type()); - return (BiFunction) callSite.getTarget().invokeExact(); - } catch (Throwable e) { - return null; - } - } - - private static BiFunction getBytesStringZeroCopyCtr() { - if (!STRING_VALUE_FIELD_IS_BYTES) { - return null; - } - MethodHandle handle = getJavaStringZeroCopyCtrHandle(); - if (handle == null) { - return null; - } - // Faster than handle.invokeExact(data, byte) - try { - MethodType instantiatedMethodType = - MethodType.methodType(handle.type().returnType(), new Class[] {byte[].class, Byte.class}); - CallSite callSite = - LambdaMetafactory.metafactory( - STRING_LOOK_UP, - "apply", - MethodType.methodType(BiFunction.class), - handle.type().generic(), - handle, - instantiatedMethodType); - return (BiFunction) callSite.getTarget().invokeExact(); - } catch (Throwable e) { - return null; - } - } - - private static Function getLatinBytesStringZeroCopyCtr() { - if (!STRING_VALUE_FIELD_IS_BYTES) { - return null; - } - if (STRING_LOOK_UP == null) { - return null; - } - try { - Class clazz = Class.forName("java.lang.StringCoding"); - MethodHandles.Lookup caller = STRING_LOOK_UP.in(clazz); - // JDK17 removed this method. - MethodHandle handle = - caller.findStatic( - clazz, "newStringLatin1", MethodType.methodType(String.class, byte[].class)); - // Faster than handle.invokeExact(data, byte) - return _JDKAccess.makeFunction(caller, handle, Function.class); - } catch (Throwable e) { - return null; - } - } - - private static MethodHandle getJavaStringZeroCopyCtrHandle() { - Preconditions.checkArgument(JdkVersion.MAJOR_VERSION >= 8); - if (STRING_LOOK_UP == null) { - return null; - } - try { - if (STRING_VALUE_FIELD_IS_CHARS) { - return STRING_LOOK_UP.findConstructor( - String.class, MethodType.methodType(void.class, char[].class, boolean.class)); - } else { - return STRING_LOOK_UP.findConstructor( - String.class, MethodType.methodType(void.class, byte[].class, byte.class)); - } - } catch (Exception e) { - return null; - } - } - private static void writeCharsUTF16BEToHeap( char[] chars, int arrIndex, int numBytes, byte[] targetArray) { // Write to heap memory then copy is 250% faster than unsafe write to direct memory. diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/URLSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/URLSerializer.java index 70043723cb..239e731eb2 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/URLSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/URLSerializer.java @@ -21,6 +21,7 @@ import java.net.MalformedURLException; import java.net.URL; +import org.apache.fory.context.CopyContext; import org.apache.fory.context.ReadContext; import org.apache.fory.context.WriteContext; import org.apache.fory.resolver.TypeResolver; @@ -28,10 +29,10 @@ /** Serializer for {@link URL}. */ // TODO(chaokunyang) ensure security to avoid dnslog detection. -public final class URLSerializer extends AbstractObjectSerializer { +public final class URLSerializer extends Serializer { public URLSerializer(TypeResolver typeResolver, Class type) { - super(typeResolver, type); + super(typeResolver.getConfig(), type); } public void write(WriteContext writeContext, URL object) { @@ -46,4 +47,14 @@ public URL read(ReadContext readContext) { throw new IllegalStateException("unreachable"); } } + + @Override + public URL copy(CopyContext copyContext, URL originURL) { + try { + return new URL(originURL.toExternalForm()); + } catch (MalformedURLException e) { + ExceptionUtils.throwException(e); + throw new IllegalStateException("unreachable"); + } + } } diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/ChildContainerSerializers.java b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/ChildContainerSerializers.java index 4a80478a31..3f7670fd47 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/ChildContainerSerializers.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/ChildContainerSerializers.java @@ -52,6 +52,7 @@ import org.apache.fory.exception.ForyException; import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.meta.TypeDef; +import org.apache.fory.reflect.ObjectCreator; import org.apache.fory.reflect.ReflectionUtils; import org.apache.fory.resolver.ClassResolver; import org.apache.fory.resolver.TypeResolver; @@ -73,6 +74,23 @@ */ @SuppressWarnings({"unchecked", "rawtypes"}) public class ChildContainerSerializers { + private static final ObjectCreator FIELD_ONLY_CREATOR = new FieldOnlyCreator(); + + private static final class FieldOnlyCreator extends ObjectCreator { + private FieldOnlyCreator() { + super(Object.class); + } + + @Override + public Object newInstance() { + throw new UnsupportedOperationException("Child-container slots do not create objects"); + } + + @Override + public Object newInstanceWithArguments(Object... arguments) { + throw new UnsupportedOperationException("Child-container slots do not create objects"); + } + } public static Class getCollectionSerializerClass(Class cls) { if (ChildCollectionSerializer.superClasses.contains(cls) @@ -626,7 +644,7 @@ private static Serializer[] buildSlotsSerializers( slotsSerializer = new CompatibleLayerSerializer(typeResolver, cls, layerTypeDef, layerMarkerClass); } else { - slotsSerializer = new ObjectSerializer<>(typeResolver, cls, false); + slotsSerializer = new ObjectSerializer<>(typeResolver, cls, false, slotCreator()); } serializers.add(slotsSerializer); cls = (Class) cls.getSuperclass(); @@ -636,6 +654,10 @@ private static Serializer[] buildSlotsSerializers( return serializers.toArray(new Serializer[0]); } + private static ObjectCreator slotCreator() { + return (ObjectCreator) FIELD_ONLY_CREATOR; + } + private static void readAndSetFields( ReadContext readContext, TypeResolver typeResolver, diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/CollectionSerializers.java b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/CollectionSerializers.java index 16a420fd97..df3e3a14f4 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/CollectionSerializers.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/CollectionSerializers.java @@ -21,7 +21,6 @@ import java.io.Externalizable; import java.lang.invoke.MethodHandle; -import java.lang.invoke.MethodHandles; import java.lang.reflect.Field; import java.util.ArrayDeque; import java.util.ArrayList; @@ -59,8 +58,7 @@ import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.memory.MemoryUtils; import org.apache.fory.platform.GraalvmSupport; -import org.apache.fory.platform.UnsafeOps; -import org.apache.fory.platform.internal._JDKAccess; +import org.apache.fory.platform.JdkVersion; import org.apache.fory.reflect.FieldAccessor; import org.apache.fory.reflect.ReflectionUtils; import org.apache.fory.resolver.ClassResolver; @@ -162,7 +160,7 @@ public void write(WriteContext writeContext, List value) { super.write(writeContext, value); } else { Object[] array = - !MemoryUtils.JDK_INTERNAL_FIELD_ACCESS || GraalvmSupport.IN_GRAALVM_NATIVE_IMAGE + !MemoryUtils.JDK_COLLECTION_FIELD_ACCESS || GraalvmSupport.IN_GRAALVM_NATIVE_IMAGE ? value.toArray() : (Object[]) ArrayAccess.ACCESSOR.getObject(value); writeContext.writeRef(array); @@ -554,17 +552,27 @@ public static final class SetFromMapSerializer extends CollectionSerializer type = Class.forName("java.util.Collections$SetFromMap"); Field mapField = type.getDeclaredField("m"); MAP_ACCESSOR = FieldAccessor.createAccessor(mapField); - MethodHandles.Lookup lookup = _JDKAccess._trustedLookup(type); - M_SETTER = lookup.findSetter(type, "m", Map.class); - S_SETTER = lookup.findSetter(type, "s", Set.class); + } catch (final Exception e) { + throw new RuntimeException(e); + } + } + } + + private static final class LegacySetFromMapAccess { + private static final FieldAccessor M_ACCESSOR; + private static final FieldAccessor S_ACCESSOR; + + static { + try { + Class type = Class.forName("java.util.Collections$SetFromMap"); + M_ACCESSOR = FieldAccessor.createAccessor(type.getDeclaredField("m")); + S_ACCESSOR = FieldAccessor.createAccessor(type.getDeclaredField("s")); } catch (final Exception e) { throw new RuntimeException(e); } @@ -588,18 +596,19 @@ public Collection newCollection(ReadContext readContext) { set = Collections.newSetFromMap(mapSerializer.newMap(readContext)); setNumElements(mapSerializer.getAndClearNumElements()); } else { - if (!MemoryUtils.JDK_INTERNAL_FIELD_ACCESS || GraalvmSupport.IN_GRAALVM_NATIVE_IMAGE) { + if (!MemoryUtils.JDK_COLLECTION_FIELD_ACCESS || GraalvmSupport.IN_GRAALVM_NATIVE_IMAGE) { throw new UnsupportedOperationException( "This runtime cannot read legacy SetFromMap payloads that require hidden JDK field " + "restoration"); } Map map = (Map) mapSerializer.read(readContext); try { - set = UnsafeOps.newInstance(type); - JvmSetFromMapAccess.M_SETTER.invoke(set, map); - JvmSetFromMapAccess.S_SETTER.invoke(set, map.keySet()); + set = Collections.newSetFromMap(new HashMap<>()); + LegacySetFromMapAccess.M_ACCESSOR.putObject(set, map); + LegacySetFromMapAccess.S_ACCESSOR.putObject(set, map.keySet()); } catch (Throwable e) { - throw new RuntimeException(e); + throw new UnsupportedOperationException( + "This runtime cannot restore legacy SetFromMap payloads through final JDK fields", e); } setNumElements(0); } @@ -610,7 +619,7 @@ public Collection newCollection(ReadContext readContext) { @Override public Collection newCollection(CopyContext copyContext, Collection originCollection) { assert !config.isXlang(); - if (!MemoryUtils.JDK_INTERNAL_FIELD_ACCESS || GraalvmSupport.IN_GRAALVM_NATIVE_IMAGE) { + if (!MemoryUtils.JDK_COLLECTION_FIELD_ACCESS || GraalvmSupport.IN_GRAALVM_NATIVE_IMAGE) { return Collections.newSetFromMap(new HashMap(originCollection.size())); } Map map = @@ -624,9 +633,9 @@ public Collection newCollection(CopyContext copyContext, Collection originCollec @Override public Collection onCollectionWrite(WriteContext writeContext, Set value) { MemoryBuffer buffer = writeContext.getBuffer(); - final Map map; - final TypeInfo typeInfo; - if (!MemoryUtils.JDK_INTERNAL_FIELD_ACCESS || GraalvmSupport.IN_GRAALVM_NATIVE_IMAGE) { + Map map; + TypeInfo typeInfo; + if (!MemoryUtils.JDK_COLLECTION_FIELD_ACCESS || GraalvmSupport.IN_GRAALVM_NATIVE_IMAGE) { HashMap source = new HashMap<>(value.size()); for (Object element : value) { source.put(element, Boolean.TRUE); @@ -638,6 +647,17 @@ public Collection onCollectionWrite(WriteContext writeContext, Set value) { typeInfo = typeResolver.getTypeInfo(map.getClass()); } MapLikeSerializer mapSerializer = (MapLikeSerializer) typeInfo.getSerializer(); + // The legacy payload restores Collections$SetFromMap by writing its final JDK fields. + // JDK25 zero-Unsafe mode cannot do that, so emit the public-constructor shape instead. + if (JdkVersion.MAJOR_VERSION >= 25 && !mapSerializer.supportCodegenHook) { + HashMap source = new HashMap<>(value.size()); + for (Object element : value) { + source.put(element, Boolean.TRUE); + } + map = source; + typeInfo = typeResolver.getTypeInfo(HashMap.class); + mapSerializer = (MapLikeSerializer) typeInfo.getSerializer(); + } typeResolver.writeTypeInfo(writeContext, typeInfo); if (mapSerializer.supportCodegenHook) { buffer.writeBoolean(true); @@ -878,7 +898,7 @@ public ArrayBlockingQueueSerializer(TypeResolver typeResolver, Class constructor; + private final Method readResolveMethod; + private final boolean biMap; + + public GuavaMapFormSerializer(TypeResolver typeResolver, Class cls, boolean biMap) { + super(typeResolver.getConfig(), cls); + this.biMap = biMap; + try { + Class mapClass = biMap ? ImmutableBiMap.class : ImmutableMap.class; + constructor = cls.getDeclaredConstructor(mapClass); + constructor.setAccessible(true); + readResolveMethod = findReadResolve(cls); + readResolveMethod.setAccessible(true); + } catch (ReflectiveOperationException e) { + throw new ForyException( + "Failed to initialize Guava serialized-form serializer for " + cls, e); + } + } + + @Override + public void write(WriteContext writeContext, Object value) { + Map map = readFormMap(value); + MemoryBuffer buffer = writeContext.getBuffer(); + buffer.writeVarUInt32Small7(map.size()); + for (Entry entry : map.entrySet()) { + writeContext.writeRef(entry.getKey()); + writeContext.writeRef(entry.getValue()); + } + } + + @Override + public Object read(ReadContext readContext) { + MemoryBuffer buffer = readContext.getBuffer(); + int size = buffer.readVarUInt32Small7(); + ImmutableMap.Builder builder = + biMap ? newImmutableBiMapBuilder(size) : newImmutableMapBuilder(size); + for (int i = 0; i < size; i++) { + builder.put(readContext.readRef(), readContext.readRef()); + } + return builder.build(); + } + + @Override + public Object copy(CopyContext copyContext, Object value) { + Map map = readFormMap(value); + ImmutableMap.Builder builder = + biMap ? newImmutableBiMapBuilder(map.size()) : newImmutableMapBuilder(map.size()); + for (Entry entry : map.entrySet()) { + Object key = entry.getKey(); + Object copyKey = key == null ? null : copyContext.copyObject(key); + Object itemValue = entry.getValue(); + Object copyValue = itemValue == null ? null : copyContext.copyObject(itemValue); + builder.put(copyKey, copyValue); + } + return newForm(builder.build()); + } + + private Map readFormMap(Object value) { + try { + return (Map) readResolveMethod.invoke(value); + } catch (ReflectiveOperationException e) { + throw new ForyException("Failed to resolve Guava serialized form " + type, e); + } + } + + private Object newForm(Map map) { + try { + Object guavaMap = biMap ? ImmutableBiMap.copyOf(map) : ImmutableMap.copyOf(map); + return constructor.newInstance(guavaMap); + } catch (ReflectiveOperationException e) { + throw new ForyException("Failed to create Guava serialized form " + type, e); + } + } + + private static Method findReadResolve(Class cls) throws NoSuchMethodException { + Class current = cls; + while (current != null) { + try { + return current.getDeclaredMethod("readResolve"); + } catch (NoSuchMethodException e) { + current = current.getSuperclass(); + } + } + throw new NoSuchMethodException(cls.getName() + ".readResolve()"); + } + } + + public static final class ImmutableIntArraySerializer extends Serializer { + + public ImmutableIntArraySerializer(TypeResolver typeResolver, Class cls) { + super(typeResolver.getConfig(), cls); + } + + @Override + public void write(WriteContext writeContext, ImmutableIntArray value) { + MemoryBuffer buffer = writeContext.getBuffer(); + int length = value.length(); + buffer.writeVarUInt32Small7(length); + for (int i = 0; i < length; i++) { + buffer.writeVarInt32(value.get(i)); + } + } + + @Override + public ImmutableIntArray read(ReadContext readContext) { + MemoryBuffer buffer = readContext.getBuffer(); + int length = buffer.readVarUInt32Small7(); + int[] values = new int[length]; + for (int i = 0; i < length; i++) { + values[i] = buffer.readVarInt32(); + } + return ImmutableIntArray.copyOf(values); + } + + @Override + public ImmutableIntArray copy(CopyContext copyContext, ImmutableIntArray value) { + return value; + } + } + + public static final class HashBasedTableSerializer extends Serializer { + + public HashBasedTableSerializer(TypeResolver typeResolver, Class cls) { + super(typeResolver.getConfig(), cls); + typeResolver.setSerializer(cls, this); + } + + @Override + public void write(WriteContext writeContext, HashBasedTable value) { + MemoryBuffer buffer = writeContext.getBuffer(); + buffer.writeVarUInt32Small7(value.size()); + for (Table.Cell cell : ((HashBasedTable) value).cellSet()) { + writeContext.writeRef(cell.getRowKey()); + writeContext.writeRef(cell.getColumnKey()); + writeContext.writeRef(cell.getValue()); + } + } + + @Override + public HashBasedTable read(ReadContext readContext) { + MemoryBuffer buffer = readContext.getBuffer(); + int size = buffer.readVarUInt32Small7(); + HashBasedTable table = HashBasedTable.create(); + if (needToWriteRef) { + readContext.setReadRef(readContext.lastPreservedRefId(), table); + } + for (int i = 0; i < size; i++) { + table.put(readContext.readRef(), readContext.readRef(), readContext.readRef()); + } + return table; + } + + @Override + public HashBasedTable copy(CopyContext copyContext, HashBasedTable value) { + HashBasedTable table = HashBasedTable.create(); + if (needToCopyRef) { + copyContext.reference(value, table); + } + for (Table.Cell cell : ((HashBasedTable) value).cellSet()) { + table.put( + copyContext.copyObject(cell.getRowKey()), + copyContext.copyObject(cell.getColumnKey()), + copyContext.copyObject(cell.getValue())); + } + return table; + } + } + public static final class ImmutableSortedMapSerializer extends MapSerializer { @@ -489,6 +675,22 @@ class GuavaEmptySortedMap {} cls = GuavaEmptySortedMap.class; resolver.registerInternalSerializer(cls, new ImmutableSortedMapSerializer(resolver, cls)); } + cls = ImmutableIntArray.class; + resolver.registerInternalSerializer(cls, new ImmutableIntArraySerializer(resolver, cls)); + cls = loadClass(IMMUTABLE_MAP_FORM_CLASS_NAME); + resolver.registerInternalSerializer(cls, new GuavaMapFormSerializer(resolver, cls, false)); + cls = loadClass(IMMUTABLE_BI_MAP_FORM_CLASS_NAME); + resolver.registerInternalSerializer(cls, new GuavaMapFormSerializer(resolver, cls, true)); + cls = HashBasedTable.class; + resolver.registerInternalSerializer(cls, new HashBasedTableSerializer(resolver, cls)); + } + + static Class loadClass(String className) { + try { + return Class.forName(className, false, GuavaCollectionSerializers.class.getClassLoader()); + } catch (ClassNotFoundException e) { + throw new RuntimeException(e); + } } static Class loadClass(String className, Class cache) { diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/ImmutableCollectionSerializers.java b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/ImmutableCollectionSerializers.java index a5b4dc539e..3eef7973c2 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/ImmutableCollectionSerializers.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/ImmutableCollectionSerializers.java @@ -32,7 +32,7 @@ import org.apache.fory.context.CopyContext; import org.apache.fory.context.ReadContext; import org.apache.fory.memory.MemoryBuffer; -import org.apache.fory.platform.AndroidSupport; +import org.apache.fory.memory.MemoryUtils; import org.apache.fory.platform.JdkVersion; import org.apache.fory.platform.internal._JDKAccess; import org.apache.fory.resolver.TypeResolver; @@ -63,7 +63,7 @@ public class ImmutableCollectionSerializers { SetN = Class.forName("java.util.ImmutableCollections$SetN"); Map1 = Class.forName("java.util.ImmutableCollections$Map1"); MapN = Class.forName("java.util.ImmutableCollections$MapN"); - if (!AndroidSupport.IS_ANDROID) { + if (MemoryUtils.JDK_COLLECTION_FIELD_ACCESS) { listFactory = _JDKAccess._trustedLookup(List.class) .findStatic(List.class, "of", MethodType.methodType(List.class, Object[].class)); @@ -79,7 +79,7 @@ public class ImmutableCollectionSerializers { .findConstructor(MapN, MethodType.methodType(void.class, Object[].class)); } } catch (ClassNotFoundException | NoSuchMethodException | IllegalAccessException e) { - if (AndroidSupport.IS_ANDROID) { + if (!MemoryUtils.JDK_COLLECTION_FIELD_ACCESS) { useStubClasses(); } else { e.printStackTrace(); @@ -144,7 +144,7 @@ public Collection copy(CopyContext copyContext, Collection originCollection) { } Object[] elements = new Object[originCollection.size()]; copyElements(copyContext, originCollection, elements); - if (AndroidSupport.IS_ANDROID) { + if (!MemoryUtils.JDK_COLLECTION_FIELD_ACCESS) { ArrayList list = new ArrayList(elements.length); Collections.addAll(list, elements); return Collections.unmodifiableList(list); @@ -160,7 +160,7 @@ public Collection copy(CopyContext copyContext, Collection originCollection) { public Collection onCollectionRead(Collection collection) { if (JdkVersion.MAJOR_VERSION > 8) { CollectionContainer container = (CollectionContainer) collection; - if (AndroidSupport.IS_ANDROID) { + if (!MemoryUtils.JDK_COLLECTION_FIELD_ACCESS) { ArrayList list = new ArrayList(container.elements.length); Collections.addAll(list, container.elements); return Collections.unmodifiableList(list); @@ -204,7 +204,7 @@ public Collection copy(CopyContext copyContext, Collection originCollection) { } Object[] elements = new Object[originCollection.size()]; copyElements(copyContext, originCollection, elements); - if (AndroidSupport.IS_ANDROID) { + if (!MemoryUtils.JDK_COLLECTION_FIELD_ACCESS) { HashSet set = new HashSet(elements.length); Collections.addAll(set, elements); return Collections.unmodifiableSet(set); @@ -220,7 +220,7 @@ public Collection copy(CopyContext copyContext, Collection originCollection) { public Collection onCollectionRead(Collection collection) { if (JdkVersion.MAJOR_VERSION > 8) { CollectionContainer container = (CollectionContainer) collection; - if (AndroidSupport.IS_ANDROID) { + if (!MemoryUtils.JDK_COLLECTION_FIELD_ACCESS) { HashSet set = new HashSet(container.elements.length); Collections.addAll(set, container.elements); return Collections.unmodifiableSet(set); @@ -265,7 +265,7 @@ public Map copy(CopyContext copyContext, Map originMap) { int size = originMap.size(); Object[] elements = new Object[size * 2]; copyEntry(copyContext, originMap, elements); - if (AndroidSupport.IS_ANDROID) { + if (!MemoryUtils.JDK_COLLECTION_FIELD_ACCESS) { return Collections.unmodifiableMap(newHashMap(elements, size)); } try { @@ -283,7 +283,7 @@ public Map copy(CopyContext copyContext, Map originMap) { public Map onMapRead(Map map) { if (JdkVersion.MAJOR_VERSION > 8) { JDKImmutableMapContainer container = (JDKImmutableMapContainer) map; - if (AndroidSupport.IS_ANDROID) { + if (!MemoryUtils.JDK_COLLECTION_FIELD_ACCESS) { return Collections.unmodifiableMap(newHashMap(container.array, container.size())); } try { diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/MapSerializers.java b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/MapSerializers.java index b94fd6f366..2adbf77c46 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/MapSerializers.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/MapSerializers.java @@ -359,7 +359,7 @@ public EnumMapSerializer(TypeResolver typeResolver) { @Override public Map onMapWrite(WriteContext writeContext, EnumMap value) { MemoryBuffer buffer = writeContext.getBuffer(); - if (!MemoryUtils.JDK_INTERNAL_FIELD_ACCESS && value.isEmpty()) { + if (!MemoryUtils.JDK_COLLECTION_FIELD_ACCESS && value.isEmpty()) { buffer.writeByte(JAVA_SERIALIZED_EMPTY_ENUM_MAP); getJavaSerializer().write(writeContext, value); return value; @@ -384,9 +384,10 @@ public EnumMap newMap(ReadContext readContext) { if (payloadMode != NORMAL_ENUM_MAP) { throw new IllegalArgumentException("Unknown EnumMap payload mode: " + payloadMode); } - setNumElements(readMapSize(buffer)); + int numElements = readMapSize(buffer); Class keyType = typeResolver.readTypeInfo(readContext).getType(); EnumMap map = new EnumMap(keyType); + setNumElements(numElements); readContext.reference(map); return map; } diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/SubListSerializers.java b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/SubListSerializers.java index fab1967e4c..f11f4b79a3 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/SubListSerializers.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/SubListSerializers.java @@ -222,7 +222,7 @@ private ViewFields( } private static ViewFields create(Class type) { - if (!MemoryUtils.JDK_INTERNAL_FIELD_ACCESS || Stub.class.isAssignableFrom(type)) { + if (!MemoryUtils.JDK_COLLECTION_FIELD_ACCESS || Stub.class.isAssignableFrom(type)) { return null; } Class cls = type; diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/SynchronizedSerializers.java b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/SynchronizedSerializers.java index b7f03987f7..a3e7893a6b 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/SynchronizedSerializers.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/SynchronizedSerializers.java @@ -93,7 +93,7 @@ public SynchronizedCollectionSerializer( @Override public void write(WriteContext writeContext, Collection object) { Preconditions.checkArgument(object.getClass() == type); - if (!MemoryUtils.JDK_INTERNAL_FIELD_ACCESS) { + if (!MemoryUtils.JDK_COLLECTION_FIELD_ACCESS) { synchronized (object) { Collection source; if (object instanceof SortedSet) { @@ -123,7 +123,7 @@ public Collection read(ReadContext readContext) { @Override public Collection copy(CopyContext copyContext, Collection object) { - if (!MemoryUtils.JDK_INTERNAL_FIELD_ACCESS) { + if (!MemoryUtils.JDK_COLLECTION_FIELD_ACCESS) { synchronized (object) { Collection mutableSource; if (object instanceof SortedSet) { @@ -159,7 +159,7 @@ public SynchronizedMapSerializer( @Override public void write(WriteContext writeContext, Map object) { Preconditions.checkArgument(object.getClass() == type); - if (!MemoryUtils.JDK_INTERNAL_FIELD_ACCESS) { + if (!MemoryUtils.JDK_COLLECTION_FIELD_ACCESS) { synchronized (object) { Map source; if (object instanceof SortedMap) { @@ -181,7 +181,7 @@ public void write(WriteContext writeContext, Map object) { @Override public Map copy(CopyContext copyContext, Map originMap) { - if (!MemoryUtils.JDK_INTERNAL_FIELD_ACCESS) { + if (!MemoryUtils.JDK_COLLECTION_FIELD_ACCESS) { synchronized (originMap) { Map mutableSource; if (originMap instanceof SortedMap) { @@ -223,7 +223,7 @@ private static Serializer createSerializer( typeResolver, factory.f0, factory.f1, - MemoryUtils.JDK_INTERNAL_FIELD_ACCESS + MemoryUtils.JDK_COLLECTION_FIELD_ACCESS ? SourceAccessors.SOURCE_COLLECTION_ACCESSOR : null); } else { @@ -231,7 +231,7 @@ private static Serializer createSerializer( typeResolver, factory.f0, factory.f1, - MemoryUtils.JDK_INTERNAL_FIELD_ACCESS ? SourceAccessors.SOURCE_MAP_ACCESSOR : null); + MemoryUtils.JDK_COLLECTION_FIELD_ACCESS ? SourceAccessors.SOURCE_MAP_ACCESSOR : null); } } diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/UnmodifiableSerializers.java b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/UnmodifiableSerializers.java index 4b0cd0372c..a024391d46 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/UnmodifiableSerializers.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/UnmodifiableSerializers.java @@ -93,7 +93,7 @@ public UnmodifiableCollectionSerializer( @Override public void write(WriteContext writeContext, Collection value) { Preconditions.checkArgument(value.getClass() == type); - if (!MemoryUtils.JDK_INTERNAL_FIELD_ACCESS) { + if (!MemoryUtils.JDK_COLLECTION_FIELD_ACCESS) { Collection source; if (value instanceof SortedSet) { source = new TreeSet(((SortedSet) value).comparator()); @@ -116,7 +116,7 @@ public Collection read(ReadContext readContext) { @Override public Collection copy(CopyContext copyContext, Collection object) { - if (!MemoryUtils.JDK_INTERNAL_FIELD_ACCESS) { + if (!MemoryUtils.JDK_COLLECTION_FIELD_ACCESS) { Collection mutableSource; if (object instanceof SortedSet) { Object comparator = copyContext.copyObject(((SortedSet) object).comparator()); @@ -149,7 +149,7 @@ public UnmodifiableMapSerializer( @Override public void write(WriteContext writeContext, Map value) { Preconditions.checkArgument(value.getClass() == type); - if (!MemoryUtils.JDK_INTERNAL_FIELD_ACCESS) { + if (!MemoryUtils.JDK_COLLECTION_FIELD_ACCESS) { Map source; if (value instanceof SortedMap) { source = new TreeMap(((SortedMap) value).comparator()); @@ -165,7 +165,7 @@ public void write(WriteContext writeContext, Map value) { @Override public Map copy(CopyContext copyContext, Map originMap) { - if (!MemoryUtils.JDK_INTERNAL_FIELD_ACCESS) { + if (!MemoryUtils.JDK_COLLECTION_FIELD_ACCESS) { Map mutableSource; if (originMap instanceof SortedMap) { Object comparator = copyContext.copyObject(((SortedMap) originMap).comparator()); @@ -203,7 +203,7 @@ private static Serializer createSerializer( typeResolver, factory.f0, factory.f1, - MemoryUtils.JDK_INTERNAL_FIELD_ACCESS + MemoryUtils.JDK_COLLECTION_FIELD_ACCESS ? SourceAccessors.SOURCE_COLLECTION_ACCESSOR : null); } else { @@ -211,7 +211,7 @@ private static Serializer createSerializer( typeResolver, factory.f0, factory.f1, - MemoryUtils.JDK_INTERNAL_FIELD_ACCESS ? SourceAccessors.SOURCE_MAP_ACCESSOR : null); + MemoryUtils.JDK_COLLECTION_FIELD_ACCESS ? SourceAccessors.SOURCE_MAP_ACCESSOR : null); } } diff --git a/java/fory-core/src/main/java/org/apache/fory/type/BFloat16.java b/java/fory-core/src/main/java/org/apache/fory/type/BFloat16.java index fb0d905ffb..b1e0f2b67e 100644 --- a/java/fory-core/src/main/java/org/apache/fory/type/BFloat16.java +++ b/java/fory-core/src/main/java/org/apache/fory/type/BFloat16.java @@ -19,6 +19,7 @@ package org.apache.fory.type; +import java.beans.ConstructorProperties; import java.io.Serializable; /** @@ -69,6 +70,7 @@ public final class BFloat16 extends Number implements Comparable, Seri private final short bits; + @ConstructorProperties("bits") private BFloat16(short bits) { this.bits = bits; } diff --git a/java/fory-core/src/main/java/org/apache/fory/type/BFloat16Array.java b/java/fory-core/src/main/java/org/apache/fory/type/BFloat16Array.java index c187e6720a..538b7095bb 100644 --- a/java/fory-core/src/main/java/org/apache/fory/type/BFloat16Array.java +++ b/java/fory-core/src/main/java/org/apache/fory/type/BFloat16Array.java @@ -19,6 +19,7 @@ package org.apache.fory.type; +import java.beans.ConstructorProperties; import java.io.Serializable; import java.util.Arrays; import java.util.Iterator; @@ -45,6 +46,11 @@ public BFloat16Array(BFloat16[] values) { } } + @ConstructorProperties("bits") + private BFloat16Array(short[] bits) { + this.bits = bits; + } + private BFloat16Array(short[] bits, boolean copy) { this.bits = copy ? Arrays.copyOf(bits, bits.length) : bits; } diff --git a/java/fory-core/src/main/java/org/apache/fory/type/Float16.java b/java/fory-core/src/main/java/org/apache/fory/type/Float16.java index c76eaa362c..562df4d180 100644 --- a/java/fory-core/src/main/java/org/apache/fory/type/Float16.java +++ b/java/fory-core/src/main/java/org/apache/fory/type/Float16.java @@ -19,6 +19,7 @@ package org.apache.fory.type; +import java.beans.ConstructorProperties; import java.io.Serializable; public final class Float16 extends Number implements Comparable, Serializable { @@ -61,6 +62,7 @@ public final class Float16 extends Number implements Comparable, Serial private final short bits; + @ConstructorProperties("bits") private Float16(short bits) { this.bits = bits; } diff --git a/java/fory-core/src/main/java/org/apache/fory/type/Float16Array.java b/java/fory-core/src/main/java/org/apache/fory/type/Float16Array.java index f6f8f3d3a5..f94e618f72 100644 --- a/java/fory-core/src/main/java/org/apache/fory/type/Float16Array.java +++ b/java/fory-core/src/main/java/org/apache/fory/type/Float16Array.java @@ -19,6 +19,7 @@ package org.apache.fory.type; +import java.beans.ConstructorProperties; import java.io.Serializable; import java.util.Arrays; import java.util.Iterator; @@ -45,6 +46,11 @@ public Float16Array(Float16[] values) { } } + @ConstructorProperties("bits") + private Float16Array(short[] bits) { + this.bits = bits; + } + private Float16Array(short[] bits, boolean copy) { this.bits = copy ? Arrays.copyOf(bits, bits.length) : bits; } diff --git a/java/fory-core/src/main/java/org/apache/fory/type/unsigned/UInt16.java b/java/fory-core/src/main/java/org/apache/fory/type/unsigned/UInt16.java index cd57b1dab1..320260816c 100644 --- a/java/fory-core/src/main/java/org/apache/fory/type/unsigned/UInt16.java +++ b/java/fory-core/src/main/java/org/apache/fory/type/unsigned/UInt16.java @@ -19,6 +19,7 @@ package org.apache.fory.type.unsigned; +import java.beans.ConstructorProperties; import java.io.Serializable; /** @@ -39,6 +40,7 @@ public final class UInt16 implements Comparable, Serializable { private final short data; + @ConstructorProperties("data") public UInt16(short data) { this.data = data; } diff --git a/java/fory-core/src/main/java/org/apache/fory/type/unsigned/UInt32.java b/java/fory-core/src/main/java/org/apache/fory/type/unsigned/UInt32.java index 2a1a4999fd..3f8052e69a 100644 --- a/java/fory-core/src/main/java/org/apache/fory/type/unsigned/UInt32.java +++ b/java/fory-core/src/main/java/org/apache/fory/type/unsigned/UInt32.java @@ -19,6 +19,7 @@ package org.apache.fory.type.unsigned; +import java.beans.ConstructorProperties; import java.io.Serializable; /** @@ -37,6 +38,7 @@ public final class UInt32 implements Comparable, Serializable { private final int data; + @ConstructorProperties("data") public UInt32(int data) { this.data = data; } diff --git a/java/fory-core/src/main/java/org/apache/fory/type/unsigned/UInt64.java b/java/fory-core/src/main/java/org/apache/fory/type/unsigned/UInt64.java index d5c7267c9f..3caff91f83 100644 --- a/java/fory-core/src/main/java/org/apache/fory/type/unsigned/UInt64.java +++ b/java/fory-core/src/main/java/org/apache/fory/type/unsigned/UInt64.java @@ -19,6 +19,7 @@ package org.apache.fory.type.unsigned; +import java.beans.ConstructorProperties; import java.io.Serializable; /** @@ -36,6 +37,7 @@ public final class UInt64 implements Comparable, Serializable { private final long data; + @ConstructorProperties("data") public UInt64(long data) { this.data = data; } diff --git a/java/fory-core/src/main/java/org/apache/fory/type/unsigned/UInt8.java b/java/fory-core/src/main/java/org/apache/fory/type/unsigned/UInt8.java index b4183108f1..70eb002787 100644 --- a/java/fory-core/src/main/java/org/apache/fory/type/unsigned/UInt8.java +++ b/java/fory-core/src/main/java/org/apache/fory/type/unsigned/UInt8.java @@ -19,6 +19,7 @@ package org.apache.fory.type.unsigned; +import java.beans.ConstructorProperties; import java.io.Serializable; /** @@ -39,6 +40,7 @@ public final class UInt8 implements Comparable, Serializable { private final byte data; + @ConstructorProperties("data") public UInt8(byte data) { this.data = data; } diff --git a/java/fory-core/src/main/java/org/apache/fory/util/DefaultValueUtils.java b/java/fory-core/src/main/java/org/apache/fory/util/DefaultValueUtils.java index 59c3088862..e5d43cc9aa 100644 --- a/java/fory-core/src/main/java/org/apache/fory/util/DefaultValueUtils.java +++ b/java/fory-core/src/main/java/org/apache/fory/util/DefaultValueUtils.java @@ -85,6 +85,10 @@ public FieldAccessor getFieldAccessor() { return fieldAccessor; } + public Class getDeclaringClass() { + return fieldAccessor == null ? null : fieldAccessor.getField().getDeclaringClass(); + } + public int getDispatchId() { return dispatchId; } @@ -414,42 +418,82 @@ private static Object convertToType(Object value, int dispatchId) { */ public static void setDefaultValues(Object obj, DefaultValueField[] defaultValueFields) { for (DefaultValueField defaultField : defaultValueFields) { - FieldAccessor fieldAccessor = defaultField.getFieldAccessor(); - if (fieldAccessor != null) { - Object defaultValue = defaultField.getDefaultValue(); - switch (defaultField.dispatchId) { - case Types.BOOL: - fieldAccessor.putBoolean(obj, (Boolean) defaultValue); - break; - case Types.INT8: - fieldAccessor.putByte(obj, (Byte) defaultValue); - break; - case Types.INT16: - fieldAccessor.putShort(obj, (Short) defaultValue); - break; - case Types.INT32: - case Types.VARINT32: - fieldAccessor.putInt(obj, (Integer) defaultValue); - break; - case Types.INT64: - case Types.VARINT64: - case Types.TAGGED_INT64: - fieldAccessor.putLong(obj, (Long) defaultValue); - break; - case Types.FLOAT32: - fieldAccessor.putFloat(obj, (Float) defaultValue); - break; - case Types.FLOAT64: - fieldAccessor.putDouble(obj, (Double) defaultValue); - break; - default: - // Object type (including String, char, boxed types not covered above) - fieldAccessor.putObject(obj, defaultValue); - } + setDefaultValue(obj, defaultField); + } + } + + public static void setDefaultValues( + Object obj, DefaultValueField[] defaultValueFields, String[] skippedFieldNames) { + setDefaultValues(obj, defaultValueFields, skippedFieldNames, null); + } + + public static void setDefaultValues( + Object obj, + DefaultValueField[] defaultValueFields, + String[] skippedFieldNames, + Class[] skippedDeclaringClasses) { + for (DefaultValueField defaultField : defaultValueFields) { + if (!contains(skippedFieldNames, skippedDeclaringClasses, defaultField)) { + setDefaultValue(obj, defaultField); } } } + private static boolean contains( + String[] values, Class[] declaringClasses, DefaultValueField defaultField) { + if (values == null) { + return false; + } + Class declaringClass = defaultField.getDeclaringClass(); + for (int i = 0; i < values.length; i++) { + if (values[i].equals(defaultField.fieldName) + && (declaringClasses == null + || i >= declaringClasses.length + || declaringClasses[i] == null + || declaringClasses[i] == declaringClass)) { + return true; + } + } + return false; + } + + private static void setDefaultValue(Object obj, DefaultValueField defaultField) { + FieldAccessor fieldAccessor = defaultField.getFieldAccessor(); + if (fieldAccessor == null) { + return; + } + Object defaultValue = defaultField.getDefaultValue(); + switch (defaultField.dispatchId) { + case Types.BOOL: + fieldAccessor.putBoolean(obj, (Boolean) defaultValue); + break; + case Types.INT8: + fieldAccessor.putByte(obj, (Byte) defaultValue); + break; + case Types.INT16: + fieldAccessor.putShort(obj, (Short) defaultValue); + break; + case Types.INT32: + case Types.VARINT32: + fieldAccessor.putInt(obj, (Integer) defaultValue); + break; + case Types.INT64: + case Types.VARINT64: + case Types.TAGGED_INT64: + fieldAccessor.putLong(obj, (Long) defaultValue); + break; + case Types.FLOAT32: + fieldAccessor.putFloat(obj, (Float) defaultValue); + break; + case Types.FLOAT64: + fieldAccessor.putDouble(obj, (Double) defaultValue); + break; + default: + // Object type (including String, char, boxed types not covered above) + fieldAccessor.putObject(obj, defaultValue); + } + } + public static Object getScalaDefaultValue(Class cls, String fieldName) { return getScalaDefaultValueSupport().getDefaultValue(cls, fieldName); } diff --git a/java/fory-core/src/main/java25/org/apache/fory/builder/CodecBuilder.java b/java/fory-core/src/main/java25/org/apache/fory/builder/CodecBuilder.java new file mode 100644 index 0000000000..b74b523c0f --- /dev/null +++ b/java/fory-core/src/main/java25/org/apache/fory/builder/CodecBuilder.java @@ -0,0 +1,732 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.builder; + +import static org.apache.fory.codegen.Expression.Invoke.inlineInvoke; +import static org.apache.fory.type.TypeUtils.CLASS_TYPE; +import static org.apache.fory.type.TypeUtils.OBJECT_ARRAY_TYPE; +import static org.apache.fory.type.TypeUtils.OBJECT_TYPE; +import static org.apache.fory.type.TypeUtils.PRIMITIVE_BOOLEAN_TYPE; +import static org.apache.fory.type.TypeUtils.PRIMITIVE_BYTE_TYPE; +import static org.apache.fory.type.TypeUtils.PRIMITIVE_CHAR_TYPE; +import static org.apache.fory.type.TypeUtils.PRIMITIVE_DOUBLE_TYPE; +import static org.apache.fory.type.TypeUtils.PRIMITIVE_FLOAT_TYPE; +import static org.apache.fory.type.TypeUtils.PRIMITIVE_INT_TYPE; +import static org.apache.fory.type.TypeUtils.PRIMITIVE_LONG_TYPE; +import static org.apache.fory.type.TypeUtils.PRIMITIVE_SHORT_TYPE; +import static org.apache.fory.type.TypeUtils.PRIMITIVE_VOID_TYPE; +import static org.apache.fory.type.TypeUtils.getRawType; + +import java.lang.invoke.MethodHandle; +import java.lang.reflect.Field; +import java.lang.reflect.Modifier; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.function.Supplier; +import java.util.stream.Collectors; +import org.apache.fory.codegen.CodegenContext; +import org.apache.fory.codegen.Expression; +import org.apache.fory.codegen.Expression.Cast; +import org.apache.fory.codegen.Expression.Inlineable; +import org.apache.fory.codegen.Expression.Invoke; +import org.apache.fory.codegen.Expression.ListExpression; +import org.apache.fory.codegen.Expression.Literal; +import org.apache.fory.codegen.Expression.Reference; +import org.apache.fory.codegen.Expression.StaticInvoke; +import org.apache.fory.collection.Tuple2; +import org.apache.fory.memory.MemoryBuffer; +import org.apache.fory.memory.NativeByteOrder; +import org.apache.fory.platform.GraalvmSupport; +import org.apache.fory.platform.UnsafeOps; +import org.apache.fory.reflect.FieldAccessor; +import org.apache.fory.reflect.ObjectCreator; +import org.apache.fory.reflect.ObjectCreators; +import org.apache.fory.reflect.ReflectionUtils; +import org.apache.fory.reflect.TypeRef; +import org.apache.fory.resolver.TypeInfo; +import org.apache.fory.resolver.TypeInfoHolder; +import org.apache.fory.type.Descriptor; +import org.apache.fory.util.Preconditions; +import org.apache.fory.util.StringUtils; +import org.apache.fory.util.function.Functions; +import org.apache.fory.util.record.RecordComponent; +import org.apache.fory.util.record.RecordUtils; + +/** + * Base builder for generating code to serialize java bean in row-format or object stream format. + * + *
    + * This builder has following requirements for the class of java bean: + *
  • public + *
  • For instance inner class, ignore outer class field. + *
  • For instance inner class, deserialized outer class field is null + *
+ */ +@SuppressWarnings("UnstableApiUsage") +public abstract class CodecBuilder { + protected static final String ROOT_OBJECT_NAME = "_f_obj"; + static TypeRef objectArrayTypeRef = TypeRef.of(Object[].class); + static TypeRef bufferTypeRef = TypeRef.of(MemoryBuffer.class); + static TypeRef classInfoTypeRef = TypeRef.of(TypeInfo.class); + static TypeRef classInfoHolderTypeRef = TypeRef.of(TypeInfoHolder.class); + + protected final CodegenContext ctx; + protected final TypeRef beanType; + protected final Class beanClass; + protected final boolean isRecord; + protected final boolean isInterface; + private final Set duplicatedFields; + public static final Reference recordComponentDefaultValues = + new Reference("recordComponentDefaultValues", OBJECT_ARRAY_TYPE); + protected final Map fieldMap = new HashMap<>(); + protected boolean recordCtrAccessible; + + public CodecBuilder(CodegenContext ctx, TypeRef beanType) { + this.ctx = ctx; + this.beanType = beanType; + this.beanClass = getRawType(beanType); + isRecord = RecordUtils.isRecord(beanClass); + isInterface = beanClass.isInterface(); + if (isRecord) { + recordCtrAccessible = recordCtrAccessible(beanClass); + } + duplicatedFields = Descriptor.getSortedDuplicatedMembers(beanClass).keySet(); + // don't ctx.addImport beanClass, because it maybe causes name collide. + ctx.reserveName(ROOT_OBJECT_NAME); + // Don't import other packages to avoid class conflicts. + // For example user class named as `Date`/`List`/`MemoryBuffer` + // Skip Java reserved words since they can't be used as variable names anyway + // (e.g., Kotlin allows field names like "new" which are valid at bytecode level) + ReflectionUtils.getFields(beanType.getRawType(), true).stream() + .map(Field::getName) + .filter(name -> !CodegenContext.JAVA_RESERVED_WORDS.contains(name)) + .collect(Collectors.toSet()) + .forEach(ctx::reserveName); + } + + public abstract String codecClassName(Class cls); + + /** Generate codec class code. */ + public abstract String genCode(); + + /** Returns an expression that serialize java bean of type {@link CodecBuilder#beanClass}. */ + public abstract Expression buildEncodeExpression(); + + protected boolean sourcePublicAccessible(Class cls) { + return ctx.sourcePublicAccessible(cls); + } + + protected boolean fieldNullable(Descriptor descriptor) { + return false; + } + + protected Expression tryInlineCast(Expression expression, TypeRef targetType) { + return tryCastIfPublic(expression, targetType, true); + } + + protected Expression tryCastIfPublic(Expression expression, TypeRef targetType) { + return tryCastIfPublic(expression, targetType, false); + } + + protected Expression tryCastIfPublic( + Expression expression, TypeRef targetType, boolean inline) { + Class rawType = getRawType(targetType); + if (inline) { + if (sourcePublicAccessible(rawType)) { + return new Cast(expression, targetType); + } else { + return new Cast(expression, ReflectionUtils.getPublicSuperType(TypeRef.of(rawType))); + } + } + return tryCastIfPublic(expression, targetType, "castedValue"); + } + + protected Expression tryCastIfPublic( + Expression expression, TypeRef targetType, String valuePrefix) { + Class rawType = getRawType(targetType); + Class expressionRawType = getRawType(expression.type()); + // Source casts use erased Java types. Captured wildcard metadata can fail full generic subtype + // checks even when the emitted local variable already has an assignable raw type. + boolean rawTypeAlreadyAssignable = rawType.isAssignableFrom(expressionRawType); + if (sourcePublicAccessible(rawType) + && !expression.type().wrap().isSubtypeOf(targetType.wrap()) + && !rawTypeAlreadyAssignable) { + return new Cast(expression, targetType, valuePrefix); + } + if (rawType.isArray()) { + return new Cast(expression, OBJECT_ARRAY_TYPE, valuePrefix); + } + return expression; + } + + protected Reference getRecordCtrHandle() { + String fieldName = "_record_ctr_"; + Reference fieldRef = fieldMap.get(fieldName); + if (fieldRef == null) { + // trigger cache for graalvm + RecordUtils.getRecordCtrHandle(beanClass); + StaticInvoke getRecordCtrHandle = + new StaticInvoke( + RecordUtils.class, + "getRecordCtrHandle", + TypeRef.of(MethodHandle.class), + beanClassExpr()); + ctx.addField(ctx.type(MethodHandle.class), fieldName, getRecordCtrHandle); + fieldRef = new Reference(fieldName, TypeRef.of(MethodHandle.class)); + fieldMap.put(fieldName, fieldRef); + } + return fieldRef; + } + + protected Expression buildDefaultComponentsArray() { + return new StaticInvoke( + UnsafeOps.class, "copyObjectArray", OBJECT_ARRAY_TYPE, recordComponentDefaultValues); + } + + /** Returns an expression that get field value from bean. */ + protected Expression getFieldValue(Expression inputBeanExpr, Descriptor descriptor) { + TypeRef fieldType = descriptor.getTypeRef(); + Class rawType = descriptor.getRawType(); + String fieldName = descriptor.getName(); + boolean fieldNullable = fieldNullable(descriptor); + if (isInterface) { + return new Invoke(inputBeanExpr, descriptor.getName(), fieldName, fieldType, fieldNullable); + } + if (isRecord) { + return getRecordFieldValue(inputBeanExpr, descriptor); + } + if (duplicatedFields.contains(fieldName) || !Modifier.isPublic(beanClass.getModifiers())) { + return unsafeAccessField(inputBeanExpr, beanClass, descriptor); + } + if (!sourcePublicAccessible(rawType)) { + fieldType = OBJECT_TYPE; + } + // public field or non-private non-java field access field directly. + if (Modifier.isPublic(descriptor.getModifiers())) { + return new Expression.FieldValue(inputBeanExpr, fieldName, fieldType, fieldNullable, false); + } else if (descriptor.getReadMethod() != null + && Modifier.isPublic(descriptor.getReadMethod().getModifiers())) { + return new Invoke( + inputBeanExpr, descriptor.getReadMethod().getName(), fieldName, fieldType, fieldNullable); + } else { + if (!Modifier.isPrivate(descriptor.getModifiers())) { + if (AccessorHelper.defineAccessor(descriptor.getField())) { + return new StaticInvoke( + AccessorHelper.getAccessorClass(descriptor.getField()), + fieldName, + fieldType, + fieldNullable, + inputBeanExpr); + } + } + if (descriptor.getReadMethod() != null + && !Modifier.isPrivate(descriptor.getReadMethod().getModifiers())) { + if (AccessorHelper.defineAccessor(descriptor.getReadMethod())) { + return new StaticInvoke( + AccessorHelper.getAccessorClass(descriptor.getReadMethod()), + descriptor.getReadMethod().getName(), + fieldType, + fieldNullable, + inputBeanExpr); + } + } + return unsafeAccessField(inputBeanExpr, beanClass, descriptor); + } + } + + private Expression getRecordFieldValue(Expression inputBeanExpr, Descriptor descriptor) { + TypeRef fieldType = descriptor.getTypeRef(); + if (!sourcePublicAccessible(descriptor.getRawType())) { + fieldType = OBJECT_TYPE; + } + String fieldName = descriptor.getName(); + boolean fieldNullable = fieldNullable(descriptor); + if (Modifier.isPublic(beanClass.getModifiers())) { + Preconditions.checkNotNull(descriptor.getReadMethod()); + return new Invoke( + inputBeanExpr, descriptor.getReadMethod().getName(), fieldName, fieldType, fieldNullable); + } else { + String key = "_" + fieldName + "_getter_"; + Reference ref = fieldMap.get(key); + Tuple2, String> methodInfo = Functions.getterMethodInfo(descriptor.getRawType()); + if (ref == null) { + Class funcInterface = methodInfo.f0; + TypeRef getterType = TypeRef.of(funcInterface); + if (GraalvmSupport.isGraalBuildTime()) { + // generate getter ahead at native image build time. + Functions.makeGetterFunction(beanClass, fieldName); + } + Expression getter = + new StaticInvoke( + Functions.class, + "makeGetterFunction", + OBJECT_TYPE, + beanClassExpr(), + Literal.ofString(fieldName)); + getter = new Cast(getter, getterType); + ctx.addField(funcInterface, key, getter); + ref = new Reference(key, getterType); + fieldMap.put(key, ref); + } + if (!fieldType.isPrimitive()) { + Expression v = inlineInvoke(ref, methodInfo.f1, OBJECT_TYPE, fieldNullable, inputBeanExpr); + return tryCastIfPublic(v, descriptor.getTypeRef(), fieldName); + } else { + return new Invoke(ref, methodInfo.f1, fieldType, fieldNullable, inputBeanExpr); + } + } + } + + /** Returns an expression that get field value> from bean using reflection. */ + private Expression reflectAccessField( + Expression inputObject, Class cls, Descriptor descriptor) { + Reference fieldRef = getReflectField(cls, descriptor.getField()); + // boolean fieldNullable = !descriptor.getTypeToken().isPrimitive(); + Invoke getObj = + new Invoke(fieldRef, "get", OBJECT_TYPE, fieldNullable(descriptor), inputObject); + return new Cast(getObj, descriptor.getTypeRef(), descriptor.getName()); + } + + /** Returns an expression that get field value> from bean using `Unsafe`. */ + private Expression unsafeAccessField( + Expression inputObject, Class cls, Descriptor descriptor) { + String fieldName = descriptor.getName(); + Reference fieldAccessor = getFieldAccessor(cls, descriptor); + boolean fieldNullable = fieldNullable(descriptor); + if (descriptor.getTypeRef().isPrimitive()) { + Preconditions.checkArgument(!fieldNullable); + TypeRef returnType = descriptor.getTypeRef(); + String funcName = "get" + StringUtils.capitalize(descriptor.getRawType().toString()); + return new Invoke(fieldAccessor, funcName, returnType, false, inputObject); + } else { + Invoke getObj = new Invoke(fieldAccessor, "getObject", OBJECT_TYPE, fieldNullable, inputObject); + return tryCastIfPublic(getObj, descriptor.getTypeRef(), fieldName); + } + } + + private Reference getFieldAccessor(Class cls, Descriptor descriptor) { + Field field = descriptor.getField(); + String fieldName = descriptor.getName(); + String fieldAccessorName = + (duplicatedFields.contains(fieldName) + ? field.getDeclaringClass().getName().replaceAll("\\.|\\$", "_") + "_" + : "") + + fieldName + + "_accessor_"; + return getOrCreateField( + true, + FieldAccessor.class, + fieldAccessorName, + () -> + new StaticInvoke( + FieldAccessor.class, + "createAccessor", + TypeRef.of(FieldAccessor.class), + getReflectField(field.getDeclaringClass(), field, false))); + } + + /** + * Returns an expression that deserialize data as a java bean of type {@link + * CodecBuilder#beanClass}. + */ + public abstract Expression buildDecodeExpression(); + + /** Returns an expression that set field value to bean. */ + protected Expression setFieldValue(Expression bean, Descriptor d, Expression value) { + String fieldName = d.getName(); + if (value instanceof Inlineable) { + ((Inlineable) value).inline(); + } + if (duplicatedFields.contains(fieldName) || !sourcePublicAccessible(beanClass)) { + return unsafeSetField(bean, d, value); + } + if (!d.isFinalField() + && Modifier.isPublic(d.getModifiers()) + && Modifier.isPublic(d.getRawType().getModifiers())) { + if (!d.getRawType().isAssignableFrom(value.type().getRawType())) { + value = tryInlineCast(value, d.getTypeRef()); + } + return new Expression.SetField(bean, fieldName, value); + } else if (d.getWriteMethod() != null && Modifier.isPublic(d.getWriteMethod().getModifiers())) { + if (!d.getRawType().isAssignableFrom(value.type().getRawType())) { + value = tryInlineCast(value, d.getTypeRef()); + } + return new Invoke(bean, d.getWriteMethod().getName(), value); + } else { + if (!d.isFinalField() && !Modifier.isPrivate(d.getModifiers())) { + if (AccessorHelper.defineSetter(d.getField())) { + Class accessorClass = AccessorHelper.getAccessorClass(d.getField()); + if (!d.getRawType().isAssignableFrom(value.type().getRawType())) { + value = tryInlineCast(value, d.getTypeRef()); + } + return new StaticInvoke( + accessorClass, d.getName(), PRIMITIVE_VOID_TYPE, false, bean, value); + } + } + if (d.getWriteMethod() != null && !Modifier.isPrivate(d.getWriteMethod().getModifiers())) { + if (AccessorHelper.defineSetter(d.getWriteMethod())) { + Class accessorClass = AccessorHelper.getAccessorClass(d.getWriteMethod()); + if (!d.getRawType().isAssignableFrom(value.type().getRawType())) { + value = tryInlineCast(value, d.getTypeRef()); + } + return new StaticInvoke( + accessorClass, d.getWriteMethod().getName(), PRIMITIVE_VOID_TYPE, false, bean, value); + } + } + return unsafeSetField(bean, d, value); + } + } + + /** + * Returns an expression that set field value to bean using reflection. + */ + private Expression reflectSetField(Expression bean, Field field, Expression value) { + // Class maybe have getter, but don't have setter, so we can't rely on reflectAccessField to + // populate fieldMap + Reference fieldRef = getReflectField(getRawType(bean.type()), field); + Preconditions.checkNotNull(fieldRef); + return new Invoke(fieldRef, "set", bean, value); + } + + /** + * Returns an expression that set field value to bean using `Unsafe`. + */ + private Expression unsafeSetField(Expression bean, Descriptor descriptor, Expression value) { + TypeRef fieldType = descriptor.getTypeRef(); + Reference fieldAccessor = getFieldAccessor(beanClass, descriptor); + if (descriptor.getTypeRef().isPrimitive()) { + Preconditions.checkArgument(getRawType(value.type()) == getRawType(fieldType)); + String funcName = "put" + StringUtils.capitalize(getRawType(fieldType).toString()); + return new Invoke(fieldAccessor, funcName, bean, value); + } else { + return new Invoke(fieldAccessor, "putObject", bean, value); + } + } + + private Reference getReflectField(Class cls, Field field) { + return getReflectField(cls, field, true); + } + + private Reference getReflectField(Class cls, Field field, boolean setAccessible) { + String fieldName = field.getName(); + String fieldRefName; + if (duplicatedFields.contains(fieldName)) { + fieldRefName = + field.getDeclaringClass().getName().replaceAll("\\.|\\$", "_") + "_" + fieldName + "_Field"; + } else { + fieldRefName = fieldName + "_Field"; + } + return getOrCreateField( + true, + Field.class, + fieldRefName, + () -> { + TypeRef fieldTypeRef = TypeRef.of(Field.class); + Class declaringClass = field.getDeclaringClass(); + Expression classExpr = + staticClassFieldExpr( + declaringClass, declaringClass.getName().replaceAll("\\.|\\$", "_") + "__class__"); + Expression fieldExpr; + if (GraalvmSupport.isGraalBuildTime()) { + fieldExpr = + inlineInvoke( + classExpr, "getDeclaredField", fieldTypeRef, Literal.ofString(fieldName)); + } else { + fieldExpr = + reflectionUtilsInvoke( + "getField", fieldTypeRef, classExpr, Literal.ofString(fieldName)); + } + if (!setAccessible) { + return fieldExpr; + } + Invoke setAccess = new Invoke(fieldExpr, "setAccessible", Literal.True); + return new ListExpression(setAccess, fieldExpr); + }); + } + + protected Reference getOrCreateField( + boolean isStatic, Class type, String fieldName, Supplier value) { + Reference fieldRef = fieldMap.get(fieldName); + if (fieldRef == null) { + fieldName = ctx.newName(fieldName); + ctx.addField(isStatic, true, ctx.type(type), fieldName, value.get()); + fieldRef = new Reference(fieldName, TypeRef.of(type)); + fieldMap.put(fieldName, fieldRef); + } + return fieldRef; + } + + /** Returns an Expression that create a new java object of type {@link CodecBuilder#beanClass}. */ + protected Expression newBean() { + // TODO allow default access-level class. + if (sourcePublicAccessible(beanClass) && ReflectionUtils.hasPublicNoArgConstructor(beanClass)) { + return new Expression.NewInstance(beanType); + } else { + ObjectCreators.getObjectCreator(beanClass); // trigger cache + Invoke newInstance = new Invoke(getObjectCreator(beanClass), "newInstance", OBJECT_TYPE); + return sourcePublicAccessible(beanClass) ? new Cast(newInstance, beanType) : newInstance; + } + } + + protected Expression getObjectCreator(Class type) { + ObjectCreators.getObjectCreator(type); // trigger cache + return getOrCreateField( + true, + ObjectCreator.class, + ctx.newName("objectCreator_" + type.getSimpleName()), + () -> + new StaticInvoke( + ObjectCreators.class, + "getObjectCreator", + TypeRef.of(ObjectCreator.class), + staticBeanClassExpr())); + } + + protected void buildRecordComponentDefaultValues() { + ctx.reserveName(recordComponentDefaultValues.name()); + StaticInvoke expr = + new StaticInvoke( + RecordUtils.class, + "buildRecordComponentDefaultValues", + OBJECT_ARRAY_TYPE, + beanClassExpr()); + ctx.addField(Object[].class, recordComponentDefaultValues.name(), expr); + } + + static boolean recordCtrAccessible(Class cls) { + // support unexported packages in module + if (!Modifier.isPublic(cls.getModifiers())) { + return false; + } + for (RecordComponent component : Objects.requireNonNull(RecordUtils.getRecordComponents(cls))) { + if (!Modifier.isPublic(component.getType().getModifiers())) { + return false; + } + } + return true; + } + + protected Expression beanClassExpr(Class cls) { + if (cls == beanClass) { + return staticBeanClassExpr(); + } + if (GraalvmSupport.isGraalBuildTime()) { + String name = cls.getName().replaceAll("\\.|\\$", "_") + "__class__"; + return getOrCreateField( + true, + Class.class, + name, + () -> + inlineReflectionUtilsInvoke( + "loadClass", CLASS_TYPE, Literal.ofString(cls.getName()))); + } + throw new UnsupportedOperationException(); + } + + protected Expression beanClassExpr() { + if (GraalvmSupport.isGraalBuildTime()) { + return staticBeanClassExpr(); + } + throw new UnsupportedOperationException(); + } + + protected Expression staticBeanClassExpr() { + if (sourcePublicAccessible(beanClass)) { + return Literal.ofClass(beanClass); + } + return staticClassFieldExpr(beanClass, "__class__"); + } + + protected Expression staticClassFieldExpr(Class cls, String fieldName) { + if (sourcePublicAccessible(cls)) { + return Literal.ofClass(cls); + } + return getOrCreateField( + true, + Class.class, + fieldName, + () -> + inlineReflectionUtilsInvoke("loadClass", CLASS_TYPE, Literal.ofString(cls.getName()))); + } + + private StaticInvoke reflectionUtilsInvoke( + String methodName, TypeRef returnType, Expression... arguments) { + return new StaticInvoke( + ReflectionUtils.class, methodName, "", returnType, false, false, false, arguments); + } + + private StaticInvoke inlineReflectionUtilsInvoke( + String methodName, TypeRef returnType, Expression... arguments) { + return new StaticInvoke( + ReflectionUtils.class, methodName, "", returnType, false, true, false, arguments); + } + + /** Build unsafePut operation. */ + protected Expression unsafePut(Expression base, Expression pos, Expression value) { + return new StaticInvoke(UnsafeOps.class, "putByte", base, pos, value); + } + + protected Expression unsafePutBoolean(Expression base, Expression pos, Expression value) { + return new StaticInvoke(UnsafeOps.class, "putBoolean", base, pos, value); + } + + protected Expression unsafePutChar(Expression base, Expression pos, Expression value) { + return new StaticInvoke(UnsafeOps.class, "putChar", base, pos, value); + } + + protected Expression unsafePutShort(Expression base, Expression pos, Expression value) { + return new StaticInvoke(UnsafeOps.class, "putShort", base, pos, value); + } + + protected Expression unsafePutInt(Expression base, Expression pos, Expression value) { + return new StaticInvoke(UnsafeOps.class, "putInt", base, pos, value); + } + + protected Expression unsafePutLong(Expression base, Expression pos, Expression value) { + return new StaticInvoke(UnsafeOps.class, "putLong", base, pos, value); + } + + protected Expression unsafePutFloat(Expression base, Expression pos, Expression value) { + return new StaticInvoke(UnsafeOps.class, "putFloat", base, pos, value); + } + + /** Build unsafePutDouble operation. */ + protected Expression unsafePutDouble(Expression base, Expression pos, Expression value) { + return new StaticInvoke(UnsafeOps.class, "putDouble", base, pos, value); + } + + /** Build unsafeGet operation. */ + protected Expression unsafeGet(Expression base, Expression pos) { + return new StaticInvoke(UnsafeOps.class, "getByte", PRIMITIVE_BYTE_TYPE, base, pos); + } + + protected Expression unsafeGetBoolean(Expression base, Expression pos) { + return new StaticInvoke(UnsafeOps.class, "getBoolean", PRIMITIVE_BOOLEAN_TYPE, base, pos); + } + + protected Expression unsafeGetChar(Expression base, Expression pos) { + StaticInvoke expr = + new StaticInvoke(UnsafeOps.class, "getChar", PRIMITIVE_CHAR_TYPE, base, pos); + if (!NativeByteOrder.IS_LITTLE_ENDIAN) { + expr = new StaticInvoke(Character.class, "reverseBytes", PRIMITIVE_CHAR_TYPE, expr.inline()); + } + return expr; + } + + protected Expression unsafeGetShort(Expression base, Expression pos) { + StaticInvoke expr = + new StaticInvoke(UnsafeOps.class, "getShort", PRIMITIVE_SHORT_TYPE, base, pos); + if (!NativeByteOrder.IS_LITTLE_ENDIAN) { + expr = new StaticInvoke(Short.class, "reverseBytes", PRIMITIVE_SHORT_TYPE, expr.inline()); + } + return expr; + } + + protected Expression unsafeGetInt(Expression base, Expression pos) { + StaticInvoke expr = new StaticInvoke(UnsafeOps.class, "getInt", PRIMITIVE_INT_TYPE, base, pos); + if (!NativeByteOrder.IS_LITTLE_ENDIAN) { + expr = new StaticInvoke(Integer.class, "reverseBytes", PRIMITIVE_INT_TYPE, expr.inline()); + } + return expr; + } + + protected Expression unsafeGetLong(Expression base, Expression pos) { + StaticInvoke expr = + new StaticInvoke(UnsafeOps.class, "getLong", PRIMITIVE_LONG_TYPE, base, pos); + if (!NativeByteOrder.IS_LITTLE_ENDIAN) { + expr = new StaticInvoke(Long.class, "reverseBytes", PRIMITIVE_LONG_TYPE, expr.inline()); + } + return expr; + } + + protected Expression unsafeGetFloat(Expression base, Expression pos) { + StaticInvoke expr = new StaticInvoke(UnsafeOps.class, "getInt", PRIMITIVE_INT_TYPE, base, pos); + if (!NativeByteOrder.IS_LITTLE_ENDIAN) { + expr = new StaticInvoke(Integer.class, "reverseBytes", PRIMITIVE_INT_TYPE, expr.inline()); + } + return new StaticInvoke(Float.class, "intBitsToFloat", PRIMITIVE_FLOAT_TYPE, expr.inline()); + } + + protected Expression unsafeGetDouble(Expression base, Expression pos) { + StaticInvoke expr = + new StaticInvoke(UnsafeOps.class, "getLong", PRIMITIVE_LONG_TYPE, base, pos); + if (!NativeByteOrder.IS_LITTLE_ENDIAN) { + expr = new StaticInvoke(Long.class, "reverseBytes", PRIMITIVE_LONG_TYPE, expr.inline()); + } + return new StaticInvoke(Double.class, "longBitsToDouble", PRIMITIVE_DOUBLE_TYPE, expr.inline()); + } + + protected Expression readChar(Expression buffer) { + return new Invoke(buffer, "readChar", PRIMITIVE_CHAR_TYPE); + } + + protected Expression readInt16(Expression buffer) { + String func = NativeByteOrder.IS_LITTLE_ENDIAN ? "_readInt16OnLE" : "_readInt16OnBE"; + return new Invoke(buffer, func, PRIMITIVE_SHORT_TYPE); + } + + protected Expression readInt32(Expression buffer) { + String func = NativeByteOrder.IS_LITTLE_ENDIAN ? "_readInt32OnLE" : "_readInt32OnBE"; + return new Invoke(buffer, func, PRIMITIVE_INT_TYPE); + } + + public static String readIntFunc() { + return NativeByteOrder.IS_LITTLE_ENDIAN ? "_readInt32OnLE" : "_readInt32OnBE"; + } + + protected Expression readVarInt32(Expression buffer) { + String func = NativeByteOrder.IS_LITTLE_ENDIAN ? "_readVarInt32OnLE" : "_readVarInt32OnBE"; + return new Invoke(buffer, func, PRIMITIVE_INT_TYPE); + } + + protected Expression readInt64(Expression buffer) { + return new Invoke(buffer, readLongFunc(), PRIMITIVE_LONG_TYPE); + } + + public static String readLongFunc() { + return NativeByteOrder.IS_LITTLE_ENDIAN ? "_readInt64OnLE" : "_readInt64OnBE"; + } + + public static String readInt16Func() { + return NativeByteOrder.IS_LITTLE_ENDIAN ? "_readInt16OnLE" : "_readInt16OnBE"; + } + + public static String readVarInt32Func() { + return NativeByteOrder.IS_LITTLE_ENDIAN ? "_readVarInt32OnLE" : "_readVarInt32OnBE"; + } + + public static String readFloat32Func() { + return NativeByteOrder.IS_LITTLE_ENDIAN ? "_readFloat32OnLE" : "_readFloat32OnBE"; + } + + public static String readFloat64Func() { + return NativeByteOrder.IS_LITTLE_ENDIAN ? "_readFloat64OnLE" : "_readFloat64OnBE"; + } + + protected Expression readFloat32(Expression buffer) { + return new Invoke(buffer, readFloat32Func(), PRIMITIVE_FLOAT_TYPE); + } + + protected Expression readFloat64(Expression buffer) { + return new Invoke(buffer, readFloat64Func(), PRIMITIVE_DOUBLE_TYPE); + } +} diff --git a/java/fory-core/src/main/java25/org/apache/fory/builder/ObjectCodecBuilder.java b/java/fory-core/src/main/java25/org/apache/fory/builder/ObjectCodecBuilder.java new file mode 100644 index 0000000000..b5c5f1ca59 --- /dev/null +++ b/java/fory-core/src/main/java25/org/apache/fory/builder/ObjectCodecBuilder.java @@ -0,0 +1,1465 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.builder; + +import static org.apache.fory.codegen.Code.LiteralValue.FalseLiteral; +import static org.apache.fory.codegen.Expression.Invoke.inlineInvoke; +import static org.apache.fory.codegen.ExpressionUtils.add; +import static org.apache.fory.codegen.ExpressionUtils.cast; +import static org.apache.fory.collection.Collections.ofHashSet; +import static org.apache.fory.type.TypeUtils.OBJECT_ARRAY_TYPE; +import static org.apache.fory.type.TypeUtils.OBJECT_TYPE; +import static org.apache.fory.type.TypeUtils.PRIMITIVE_BOOLEAN_TYPE; +import static org.apache.fory.type.TypeUtils.PRIMITIVE_BYTE_ARRAY_TYPE; +import static org.apache.fory.type.TypeUtils.PRIMITIVE_BYTE_TYPE; +import static org.apache.fory.type.TypeUtils.PRIMITIVE_CHAR_TYPE; +import static org.apache.fory.type.TypeUtils.PRIMITIVE_DOUBLE_TYPE; +import static org.apache.fory.type.TypeUtils.PRIMITIVE_FLOAT_TYPE; +import static org.apache.fory.type.TypeUtils.PRIMITIVE_INT_TYPE; +import static org.apache.fory.type.TypeUtils.PRIMITIVE_LONG_TYPE; +import static org.apache.fory.type.TypeUtils.PRIMITIVE_SHORT_TYPE; +import static org.apache.fory.type.TypeUtils.PRIMITIVE_VOID_TYPE; +import static org.apache.fory.type.TypeUtils.SHORT_TYPE; +import static org.apache.fory.type.TypeUtils.getRawType; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.IdentityHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.SortedMap; +import java.util.TreeMap; +import org.apache.fory.Fory; +import org.apache.fory.codegen.Code; +import org.apache.fory.codegen.CodegenContext; +import org.apache.fory.codegen.Expression; +import org.apache.fory.codegen.Expression.Cast; +import org.apache.fory.codegen.Expression.Inlineable; +import org.apache.fory.codegen.Expression.Invoke; +import org.apache.fory.codegen.Expression.ListExpression; +import org.apache.fory.codegen.Expression.Literal; +import org.apache.fory.codegen.Expression.NewInstance; +import org.apache.fory.codegen.Expression.Reference; +import org.apache.fory.codegen.Expression.ReplaceStub; +import org.apache.fory.codegen.Expression.StaticInvoke; +import org.apache.fory.codegen.ExpressionVisitor; +import org.apache.fory.logging.Logger; +import org.apache.fory.logging.LoggerFactory; +import org.apache.fory.meta.TypeDef; +import org.apache.fory.platform.UnsafeOps; +import org.apache.fory.reflect.ObjectCreator; +import org.apache.fory.reflect.ObjectCreators; +import org.apache.fory.reflect.TypeRef; +import org.apache.fory.serializer.ObjectSerializer; +import org.apache.fory.serializer.AbstractObjectSerializer; +import org.apache.fory.type.BFloat16; +import org.apache.fory.type.Descriptor; +import org.apache.fory.type.DescriptorGrouper; +import org.apache.fory.type.DispatchId; +import org.apache.fory.type.Float16; +import org.apache.fory.type.TypeUtils; +import org.apache.fory.type.Types; +import org.apache.fory.util.StringUtils; +import org.apache.fory.util.function.SerializableSupplier; +import org.apache.fory.util.record.RecordUtils; + +/** + * Generate sequential read/write code for java serialization to speed up performance. It also + * reduces space overhead introduced by aligning. Codegen only for time-consuming field, others + * delegate to fory. + * + *

In order to improve jit-compile and inline, serialization code should be spilt groups to avoid + * huge/big methods. + * + *

With meta context share enabled and compatible mode, this serializer will take all non-inner + * final types as non-final, so that fory can write class definition when write class info for those + * types. + * + * @see ObjectCodecOptimizer for code stats and split heuristics. + */ +public class ObjectCodecBuilder extends BaseObjectCodecBuilder { + private static final Logger LOG = LoggerFactory.getLogger(ObjectCodecBuilder.class); + + private final Literal classVersionHash; + protected ObjectCodecOptimizer objectCodecOptimizer; + protected Map recordReversedMapping; + protected Map fieldIndexes; + protected int[] constructorFieldIndexes; + protected boolean[] constructorFieldMask; + protected Class[] constructorFieldTypes; + + public ObjectCodecBuilder(Class beanClass, Fory fory) { + super(TypeRef.of(beanClass), fory, Generated.GeneratedObjectSerializer.class); + Collection descriptors; + DescriptorGrouper grouper; + boolean shareMeta = fory.getConfig().isMetaShareEnabled(); + if (shareMeta) { + TypeDef typeDef = typeResolver(r -> r.getTypeDef(beanClass, true)); + descriptors = typeResolver(r -> typeDef.getDescriptors(r, beanClass)); + grouper = typeResolver(r -> r.createDescriptorGrouper(typeDef, beanClass)); + } else { + grouper = typeResolver(r -> r.getFieldDescriptorGrouper(beanClass, true, false)); + descriptors = grouper.getSortedDescriptors(); + } + if (org.apache.fory.util.Utils.DEBUG_OUTPUT_ENABLED) { + LOG.info( + "========== {} sorted descriptors for {} ==========", + descriptors.size(), + beanClass.getSimpleName()); + List sortedDescriptors = grouper.getSortedDescriptors(); + for (Descriptor d : sortedDescriptors) { + LOG.info( + " {} -> {}, ref {}, nullable {}", + StringUtils.toSnakeCase(d.getName()), + d.getTypeName(), + d.isTrackingRef(), + d.isNullable()); + } + } + classVersionHash = + typeResolver.checkClassVersion() + ? new Literal( + ObjectSerializer.computeStructHash(typeResolver, grouper), PRIMITIVE_INT_TYPE) + : null; + objectCodecOptimizer = new ObjectCodecOptimizer(beanClass, grouper, false, ctx); + if (isRecord) { + if (!recordCtrAccessible) { + buildRecordComponentDefaultValues(); + } + recordReversedMapping = RecordUtils.buildFieldToComponentMapping(beanClass); + } else { + initConstructorFields(grouper.getSortedDescriptors(), true); + } + } + + protected ObjectCodecBuilder(TypeRef beanType, Fory fory, Class superSerializerClass) { + super(beanType, fory, superSerializerClass); + this.classVersionHash = null; + if (isRecord) { + if (!recordCtrAccessible) { + buildRecordComponentDefaultValues(); + } + recordReversedMapping = RecordUtils.buildFieldToComponentMapping(beanClass); + } + } + + protected final void initConstructorFields( + List sortedDescriptors, boolean allowMissingNonFinal) { + initConstructorFields(sortedDescriptors, allowMissingNonFinal, null); + } + + protected final void initConstructorFields( + List sortedDescriptors, boolean allowMissingNonFinal, String[] defaultFields) { + initConstructorFields(sortedDescriptors, allowMissingNonFinal, defaultFields, null); + } + + protected final void initConstructorFields( + List sortedDescriptors, + boolean allowMissingNonFinal, + String[] defaultFields, + Class[] defaultDeclaringClasses) { + ObjectCreator objectCreator = ObjectCreators.getObjectCreator(beanClass); + if (!objectCreator.hasConstructorFields()) { + return; + } + fieldIndexes = buildFieldIndexes(sortedDescriptors); + constructorFieldTypes = objectCreator.getConstructorFieldTypes(); + constructorFieldIndexes = + buildConstructorFieldIndexes( + sortedDescriptors, + objectCreator, + allowMissingNonFinal, + defaultFields, + defaultDeclaringClasses); + constructorFieldMask = buildConstructorFieldMask(sortedDescriptors.size()); + } + + @Override + protected String codecSuffix() { + return ""; + } + + @Override + protected void addCommonImports() { + super.addCommonImports(); + ctx.addImport(Generated.GeneratedObjectSerializer.class); + } + + /** + * Return an expression that serialize java bean of type {@link CodecBuilder#beanClass} to buffer. + */ + @Override + public Expression buildEncodeExpression() { + Reference inputObject = new Reference(ROOT_OBJECT_NAME, OBJECT_TYPE, false); + Reference buffer = new Reference(BUFFER_NAME, bufferTypeRef, false); + + ListExpression expressions = new ListExpression(); + Expression bean = tryCastIfPublic(inputObject, beanType, ctx.newName(beanClass)); + expressions.add(bean); + if (typeResolver.checkClassVersion()) { + expressions.add(new Invoke(buffer, "writeInt32", classVersionHash)); + } + expressions.addAll(serializePrimitives(bean, buffer, objectCodecOptimizer.primitiveGroups)); + int numGroups = getNumGroups(objectCodecOptimizer); + addGroupExpressions( + objectCodecOptimizer.boxedWriteGroups, numGroups, expressions, bean, buffer); + addGroupExpressions( + objectCodecOptimizer.nonPrimitiveWriteGroups, numGroups, expressions, bean, buffer); + return expressions; + } + + private void addGroupExpressions( + List> writeGroup, + int numGroups, + ListExpression expressions, + Expression bean, + Reference buffer) { + for (List group : writeGroup) { + if (group.isEmpty()) { + continue; + } + boolean inline = hasFewFields() || (group.size() == 1 && numGroups < 10); + expressions.add(serializeGroup(group, bean, buffer, inline)); + } + } + + protected boolean hasFewFields() { + return objectCodecOptimizer.descriptorGrouper.getNumDescriptors() < 6; + } + + protected int getNumGroups(ObjectCodecOptimizer objectCodecOptimizer) { + return objectCodecOptimizer.boxedWriteGroups.size() + + objectCodecOptimizer.nonPrimitiveWriteGroups.size(); + } + + private static Map buildFieldIndexes(List descriptors) { + Map indexes = new IdentityHashMap<>(); + for (int i = 0; i < descriptors.size(); i++) { + indexes.put(descriptors.get(i), i); + } + return indexes; + } + + private int[] buildConstructorFieldIndexes( + List descriptors, + ObjectCreator objectCreator, + boolean allowMissingNonFinal, + String[] defaultFields, + Class[] defaultDeclaringClasses) { + String[] names = objectCreator.getConstructorFieldNames(); + Class[] declaringClasses = objectCreator.getConstructorFieldDeclaringClasses(); + boolean[] finalFields = objectCreator.getConstructorFieldFinal(); + int[] indexes = new int[names.length]; + for (int i = 0; i < names.length; i++) { + Class declaringClass = declaringClasses == null ? null : declaringClasses[i]; + boolean allowMissing = + (allowMissingNonFinal && !finalFields[i]) + || contains(defaultFields, defaultDeclaringClasses, names[i], declaringClass); + indexes[i] = + constructorFieldIndex(descriptors, declaringClass, names[i], allowMissing); + } + return indexes; + } + + private static boolean contains( + String[] values, Class[] declaringClasses, String value, Class declaringClass) { + if (values == null) { + return false; + } + for (int i = 0; i < values.length; i++) { + if (values[i].equals(value) + && (declaringClasses == null + || i >= declaringClasses.length + || declaringClasses[i] == null + || declaringClasses[i] == declaringClass)) { + return true; + } + } + return false; + } + + private int constructorFieldIndex( + List descriptors, + Class declaringClass, + String fieldName, + boolean allowMissing) { + int index = -1; + for (int i = 0; i < descriptors.size(); i++) { + Descriptor descriptor = descriptors.get(i); + if (!descriptor.getName().equals(fieldName) + || (declaringClass != null + && (descriptor.getField() == null + || descriptor.getField().getDeclaringClass() != declaringClass))) { + continue; + } + if (index >= 0) { + throw new IllegalStateException( + "Constructor field " + fieldName + " is ambiguous for " + beanClass); + } + index = i; + } + if (index < 0) { + if (allowMissing) { + return -1; + } + throw new IllegalStateException( + "Constructor field " + fieldName + " is not serialized for " + beanClass); + } + return index; + } + + private boolean[] buildConstructorFieldMask(int size) { + boolean[] mask = new boolean[size]; + for (int index : constructorFieldIndexes) { + if (index >= 0) { + mask[index] = true; + } + } + return mask; + } + + private Expression serializeGroup( + List group, Expression bean, Expression buffer, boolean inline) { + SerializableSupplier expressionSupplier = + () -> { + ListExpression groupExpressions = new ListExpression(); + for (Descriptor d : group) { + // `bean` will be replaced by `Reference` to cut-off expr dependency. + Expression fieldValue = getFieldValue(bean, d); + walkPath.add(d.getDeclaringClass() + d.getName()); + Expression fieldExpr = serializeField(fieldValue, buffer, d); + walkPath.removeLast(); + groupExpressions.add(fieldExpr); + } + return groupExpressions; + }; + if (inline) { + return expressionSupplier.get(); + } + return objectCodecOptimizer.invokeGenerated( + writeCutPoints(bean, buffer), expressionSupplier.get(), "writeFields"); + } + + /** + * Return a list of expressions that serialize all primitive fields. This can reduce unnecessary + * grow call and increment writerIndex in writeXXX. + */ + private List serializePrimitives( + Expression bean, Expression buffer, List> primitiveGroups) { + int totalSize = getTotalSizeOfPrimitives(primitiveGroups); + if (totalSize == 0) { + return new ArrayList<>(); + } + if (config.compressInt() || config.compressLong()) { + return serializePrimitivesCompressed(bean, buffer, primitiveGroups, totalSize); + } else { + return serializePrimitivesUnCompressed(bean, buffer, primitiveGroups, totalSize); + } + } + + protected int getNumPrimitiveFields(List> primitiveGroups) { + return primitiveGroups.stream().mapToInt(List::size).sum(); + } + + private List serializePrimitivesUnCompressed( + Expression bean, Expression buffer, List> primitiveGroups, int totalSize) { + List expressions = new ArrayList<>(); + int numPrimitiveFields = getNumPrimitiveFields(primitiveGroups); + Literal totalSizeLiteral = new Literal(totalSize, PRIMITIVE_INT_TYPE); + // After this grow, following writes can be unsafe without checks. + expressions.add(new Invoke(buffer, "grow", totalSizeLiteral)); + Expression writerIndex = + new Invoke(buffer, "writerIndex", "writerIndex", PRIMITIVE_INT_TYPE); + expressions.add(writerIndex); + int acc = 0; + for (List group : primitiveGroups) { + ListExpression groupExpressions = new ListExpression(); + // use Reference to cut-off expr dependency. + for (Descriptor descriptor : group) { + int dispatchId = getNumericDescriptorDispatchId(descriptor); + // `bean` will be replaced by `Reference` to cut-off expr dependency. + Expression fieldValue = getFieldValue(bean, descriptor); + if (fieldValue instanceof Inlineable) { + ((Inlineable) fieldValue).inline(); + } + if (dispatchId == DispatchId.BOOL) { + groupExpressions.add(bufferPutBoolean(buffer, getBufferIndex(writerIndex, acc), fieldValue)); + acc += 1; + } else if (dispatchId == DispatchId.INT8) { + groupExpressions.add(bufferPutByte(buffer, getBufferIndex(writerIndex, acc), fieldValue)); + acc += 1; + } else if (dispatchId == DispatchId.UINT8) { + groupExpressions.add( + bufferPutByte( + buffer, getBufferIndex(writerIndex, acc), primitiveByteValue(fieldValue, descriptor))); + acc += 1; + } else if (dispatchId == DispatchId.CHAR) { + groupExpressions.add(bufferPutChar(buffer, getBufferIndex(writerIndex, acc), fieldValue)); + acc += 2; + } else if (dispatchId == DispatchId.INT16) { + groupExpressions.add(bufferPutInt16(buffer, getBufferIndex(writerIndex, acc), fieldValue)); + acc += 2; + } else if (dispatchId == DispatchId.UINT16) { + groupExpressions.add( + bufferPutInt16( + buffer, + getBufferIndex(writerIndex, acc), + primitiveShortValue(fieldValue, descriptor))); + acc += 2; + } else if (dispatchId == DispatchId.FLOAT16 || dispatchId == DispatchId.BFLOAT16) { + groupExpressions.add( + bufferPutInt16( + buffer, + getBufferIndex(writerIndex, acc), + new Invoke(fieldValue, "toBits", SHORT_TYPE))); + acc += 2; + } else if (dispatchId == DispatchId.INT32) { + groupExpressions.add(bufferPutInt32(buffer, getBufferIndex(writerIndex, acc), fieldValue)); + acc += 4; + } else if (dispatchId == DispatchId.UINT32) { + groupExpressions.add( + bufferPutInt32( + buffer, getBufferIndex(writerIndex, acc), primitiveIntValue(fieldValue, descriptor))); + acc += 4; + } else if (dispatchId == DispatchId.INT64 || dispatchId == DispatchId.UINT64) { + groupExpressions.add(bufferPutInt64(buffer, getBufferIndex(writerIndex, acc), fieldValue)); + acc += 8; + } else if (dispatchId == DispatchId.FLOAT32) { + groupExpressions.add(bufferPutFloat32(buffer, getBufferIndex(writerIndex, acc), fieldValue)); + acc += 4; + } else if (dispatchId == DispatchId.FLOAT64) { + groupExpressions.add(bufferPutFloat64(buffer, getBufferIndex(writerIndex, acc), fieldValue)); + acc += 8; + } else { + throw new IllegalStateException("Unsupported dispatchId: " + dispatchId); + } + } + if (hasFewFields() || numPrimitiveFields < 4) { + expressions.add(groupExpressions); + } else { + expressions.add( + objectCodecOptimizer.invokeGenerated( + ofHashSet(bean, buffer, writerIndex), groupExpressions, "writeFields")); + } + } + Expression increaseWriterIndex = + new Invoke( + buffer, + "_increaseWriterIndexUnsafe", + new Literal(totalSizeLiteral, PRIMITIVE_INT_TYPE)); + expressions.add(increaseWriterIndex); + return expressions; + } + + private List serializePrimitivesCompressed( + Expression bean, Expression buffer, List> primitiveGroups, int totalSize) { + List expressions = new ArrayList<>(); + // int/long may need extra one-byte for writing. + int extraSize = 0; + for (List group : primitiveGroups) { + for (Descriptor d : group) { + int id = getNumericDescriptorDispatchId(d); + if (id == DispatchId.INT32 + || id == DispatchId.VARINT32 + || id == DispatchId.VAR_UINT32 + || id == DispatchId.UINT32) { + // varint may be written as 5bytes, use 8bytes for written as long to reduce cost. + extraSize += 4; + } else if (id == DispatchId.INT64 + || id == DispatchId.VARINT64 + || id == DispatchId.TAGGED_INT64 + || id == DispatchId.VAR_UINT64 + || id == DispatchId.TAGGED_UINT64 + || id == DispatchId.UINT64) { + extraSize += 1; // long use 1~9 bytes. + } + } + } + int growSize = totalSize + extraSize; + // After this grow, following writes can be unsafe without checks. + expressions.add(new Invoke(buffer, "grow", Literal.ofInt(growSize))); + int numPrimitiveFields = getNumPrimitiveFields(primitiveGroups); + for (List group : primitiveGroups) { + ListExpression groupExpressions = new ListExpression(); + Expression writerIndex = + new Invoke(buffer, "writerIndex", "writerIndex", PRIMITIVE_INT_TYPE); + // use Reference to cut-off expr dependency. + int acc = 0; + boolean compressStarted = false; + for (Descriptor descriptor : group) { + int dispatchId = getNumericDescriptorDispatchId(descriptor); + // `bean` will be replaced by `Reference` to cut-off expr dependency. + Expression fieldValue = getFieldValue(bean, descriptor); + if (fieldValue instanceof Inlineable) { + ((Inlineable) fieldValue).inline(); + } + if (dispatchId == DispatchId.BOOL) { + groupExpressions.add(bufferPutBoolean(buffer, getBufferIndex(writerIndex, acc), fieldValue)); + acc += 1; + } else if (dispatchId == DispatchId.INT8) { + groupExpressions.add(bufferPutByte(buffer, getBufferIndex(writerIndex, acc), fieldValue)); + acc += 1; + } else if (dispatchId == DispatchId.UINT8) { + groupExpressions.add( + bufferPutByte( + buffer, getBufferIndex(writerIndex, acc), primitiveByteValue(fieldValue, descriptor))); + acc += 1; + } else if (dispatchId == DispatchId.CHAR) { + groupExpressions.add(bufferPutChar(buffer, getBufferIndex(writerIndex, acc), fieldValue)); + acc += 2; + } else if (dispatchId == DispatchId.INT16) { + groupExpressions.add(bufferPutInt16(buffer, getBufferIndex(writerIndex, acc), fieldValue)); + acc += 2; + } else if (dispatchId == DispatchId.UINT16) { + groupExpressions.add( + bufferPutInt16( + buffer, + getBufferIndex(writerIndex, acc), + primitiveShortValue(fieldValue, descriptor))); + acc += 2; + } else if (dispatchId == DispatchId.FLOAT16 || dispatchId == DispatchId.BFLOAT16) { + groupExpressions.add( + bufferPutInt16( + buffer, + getBufferIndex(writerIndex, acc), + new Invoke(fieldValue, "toBits", SHORT_TYPE))); + acc += 2; + } else if (dispatchId == DispatchId.FLOAT32) { + groupExpressions.add(bufferPutFloat32(buffer, getBufferIndex(writerIndex, acc), fieldValue)); + acc += 4; + } else if (dispatchId == DispatchId.FLOAT64) { + groupExpressions.add(bufferPutFloat64(buffer, getBufferIndex(writerIndex, acc), fieldValue)); + acc += 8; + } else if (dispatchId == DispatchId.INT32) { + groupExpressions.add(bufferPutInt32(buffer, getBufferIndex(writerIndex, acc), fieldValue)); + acc += 4; + } else if (dispatchId == DispatchId.UINT32) { + groupExpressions.add( + bufferPutInt32( + buffer, getBufferIndex(writerIndex, acc), primitiveIntValue(fieldValue, descriptor))); + acc += 4; + } else if (dispatchId == DispatchId.INT64 || dispatchId == DispatchId.UINT64) { + groupExpressions.add(bufferPutInt64(buffer, getBufferIndex(writerIndex, acc), fieldValue)); + acc += 8; + } else if (dispatchId == DispatchId.VARINT32) { + if (!compressStarted) { + addIncWriterIndexExpr(groupExpressions, buffer, acc); + compressStarted = true; + } + groupExpressions.add(new Invoke(buffer, "_unsafeWriteVarInt32", fieldValue)); + } else if (dispatchId == DispatchId.VAR_UINT32) { + if (!compressStarted) { + addIncWriterIndexExpr(groupExpressions, buffer, acc); + compressStarted = true; + } + groupExpressions.add( + new Invoke( + buffer, "_unsafeWriteVarUInt32", primitiveIntValue(fieldValue, descriptor))); + } else if (dispatchId == DispatchId.VARINT64) { + if (!compressStarted) { + addIncWriterIndexExpr(groupExpressions, buffer, acc); + compressStarted = true; + } + groupExpressions.add(new Invoke(buffer, "writeVarInt64", fieldValue)); + } else if (dispatchId == DispatchId.TAGGED_INT64) { + if (!compressStarted) { + addIncWriterIndexExpr(groupExpressions, buffer, acc); + compressStarted = true; + } + groupExpressions.add(new Invoke(buffer, "writeTaggedInt64", fieldValue)); + } else if (dispatchId == DispatchId.VAR_UINT64) { + if (!compressStarted) { + addIncWriterIndexExpr(groupExpressions, buffer, acc); + compressStarted = true; + } + groupExpressions.add(new Invoke(buffer, "writeVarUInt64", fieldValue)); + } else if (dispatchId == DispatchId.TAGGED_UINT64) { + if (!compressStarted) { + addIncWriterIndexExpr(groupExpressions, buffer, acc); + compressStarted = true; + } + groupExpressions.add(new Invoke(buffer, "writeTaggedUInt64", fieldValue)); + } else { + throw new IllegalStateException("Unsupported dispatchId: " + dispatchId); + } + } + if (!compressStarted) { + // int/long are sorted in the last. + addIncWriterIndexExpr(groupExpressions, buffer, acc); + } + if (hasFewFields() || numPrimitiveFields < 4) { + expressions.add(groupExpressions); + } else { + expressions.add( + objectCodecOptimizer.invokeGenerated( + ofHashSet(bean, buffer, writerIndex), groupExpressions, "writeFields")); + } + } + return expressions; + } + + private Expression bufferPutByte(Expression buffer, Expression index, Expression value) { + return new Invoke(buffer, "_unsafePutByte", index, value); + } + + private Expression bufferPutBoolean(Expression buffer, Expression index, Expression value) { + return new Invoke(buffer, "_unsafePutBoolean", index, value); + } + + private Expression bufferPutChar(Expression buffer, Expression index, Expression value) { + return new Invoke(buffer, "_unsafePutChar", index, value); + } + + private Expression bufferPutInt16(Expression buffer, Expression index, Expression value) { + return new Invoke(buffer, "_unsafePutInt16", index, value); + } + + private Expression bufferPutInt32(Expression buffer, Expression index, Expression value) { + return new Invoke(buffer, "_unsafePutInt32", index, value); + } + + private Expression bufferPutInt64(Expression buffer, Expression index, Expression value) { + return new Invoke(buffer, "_unsafePutInt64", index, value); + } + + private Expression bufferPutFloat32(Expression buffer, Expression index, Expression value) { + return bufferPutInt32( + buffer, + index, + new StaticInvoke(Float.class, "floatToRawIntBits", PRIMITIVE_INT_TYPE, value)); + } + + private Expression bufferPutFloat64(Expression buffer, Expression index, Expression value) { + return bufferPutInt64( + buffer, + index, + new StaticInvoke(Double.class, "doubleToRawLongBits", PRIMITIVE_LONG_TYPE, value)); + } + + private Expression bufferGetByte(Expression buffer, Expression index) { + return new Invoke(buffer, "_unsafeGetByte", PRIMITIVE_BYTE_TYPE, index); + } + + private Expression bufferGetBoolean(Expression buffer, Expression index) { + return new Invoke(buffer, "_unsafeGetBoolean", PRIMITIVE_BOOLEAN_TYPE, index); + } + + private Expression bufferGetChar(Expression buffer, Expression index) { + return new Invoke(buffer, "_unsafeGetChar", PRIMITIVE_CHAR_TYPE, index); + } + + private Expression bufferGetInt16(Expression buffer, Expression index) { + return new Invoke(buffer, "_unsafeGetInt16", PRIMITIVE_SHORT_TYPE, index); + } + + private Expression bufferGetInt32(Expression buffer, Expression index) { + return new Invoke(buffer, "_unsafeGetInt32", PRIMITIVE_INT_TYPE, index); + } + + private Expression bufferGetInt64(Expression buffer, Expression index) { + return new Invoke(buffer, "_unsafeGetInt64", PRIMITIVE_LONG_TYPE, index); + } + + private Expression bufferGetFloat32(Expression buffer, Expression index) { + return new StaticInvoke( + Float.class, "intBitsToFloat", PRIMITIVE_FLOAT_TYPE, bufferGetInt32(buffer, index)); + } + + private Expression bufferGetFloat64(Expression buffer, Expression index) { + return new StaticInvoke( + Double.class, + "longBitsToDouble", + PRIMITIVE_DOUBLE_TYPE, + bufferGetInt64(buffer, index)); + } + + private Expression primitiveByteValue(Expression fieldValue, Descriptor descriptor) { + return fieldValue.type().isPrimitive() + ? cast(fieldValue, PRIMITIVE_BYTE_TYPE) + : new Invoke(boxedNumericValue(fieldValue, descriptor), "byteValue", PRIMITIVE_BYTE_TYPE); + } + + private Expression primitiveShortValue(Expression fieldValue, Descriptor descriptor) { + return fieldValue.type().isPrimitive() + ? cast(fieldValue, PRIMITIVE_SHORT_TYPE) + : new Invoke(boxedNumericValue(fieldValue, descriptor), "shortValue", PRIMITIVE_SHORT_TYPE); + } + + private Expression primitiveIntValue(Expression fieldValue, Descriptor descriptor) { + return fieldValue.type().isPrimitive() + ? cast(fieldValue, PRIMITIVE_INT_TYPE) + : new Invoke(boxedNumericValue(fieldValue, descriptor), "intValue", PRIMITIVE_INT_TYPE); + } + + private Expression boxedNumericValue(Expression fieldValue, Descriptor descriptor) { + return Number.class.isAssignableFrom(getRawType(fieldValue.type())) + ? fieldValue + : cast(fieldValue, descriptor.getTypeRef()); + } + + private void addIncWriterIndexExpr(ListExpression expressions, Expression buffer, int diff) { + if (diff != 0) { + expressions.add(new Invoke(buffer, "_increaseWriterIndexUnsafe", Literal.ofInt(diff))); + } + } + + private int getTotalSizeOfPrimitives(List> primitiveGroups) { + return primitiveGroups.stream() + .flatMap(Collection::stream) + .mapToInt( + d -> { + Class rawType = d.getRawType(); + if (TypeUtils.isPrimitive(rawType) || TypeUtils.isBoxed(rawType)) { + return TypeUtils.getSizeOfPrimitiveType(TypeUtils.unwrap(rawType)); + } + return Types.getPrimitiveTypeSize(Types.getDescriptorTypeId(typeResolver, d)); + }) + .sum(); + } + + private Expression getWriterPos(Expression writerPos, long acc) { + if (acc == 0) { + return writerPos; + } + return add(writerPos, Literal.ofLong(acc)); + } + + public Expression buildDecodeExpression() { + Reference buffer = new Reference(BUFFER_NAME, bufferTypeRef, false); + ListExpression expressions = new ListExpression(); + if (typeResolver.checkClassVersion()) { + expressions.add(checkClassVersion(buffer)); + } + if (!isRecord && constructorFieldIndexes != null) { + return buildConstructorDecodeExpression(buffer, expressions); + } + Expression bean; + if (!isRecord) { + if (constructorFieldIndexes == null) { + bean = newBean(); + Expression referenceObject = invokeReadContext("reference", bean); + expressions.add(bean); + expressions.add(referenceObject); + } else { + bean = new FieldsArray(fieldIndexes.size()); + expressions.add(bean); + } + } else { + if (recordCtrAccessible) { + bean = new FieldsCollector(); + } else { + bean = buildComponentsArray(); + } + } + expressions.addAll(deserializePrimitives(bean, buffer, objectCodecOptimizer.primitiveGroups)); + int numGroups = getNumGroups(objectCodecOptimizer); + deserializeReadGroup( + objectCodecOptimizer.boxedReadGroups, numGroups, expressions, bean, buffer); + deserializeReadGroup( + objectCodecOptimizer.nonPrimitiveReadGroups, numGroups, expressions, bean, buffer); + if (isRecord) { + if (recordCtrAccessible) { + assert bean instanceof FieldsCollector; + FieldsCollector collector = (FieldsCollector) bean; + bean = createRecord(collector.recordValuesMap); + } else { + ObjectCreators.getObjectCreator(beanClass); // trigger cache and make error raised early + bean = + new Invoke(getObjectCreator(beanClass), "newInstanceWithArguments", OBJECT_TYPE, bean); + } + } + expressions.add(new Expression.Return(bean)); + return expressions; + } + + private Expression buildConstructorDecodeExpression( + Reference buffer, ListExpression expressions) { + FieldsArray fieldsArray = new FieldsArray(fieldIndexes.size()); + expressions.add(fieldsArray); + expressions.add( + new StaticInvoke( + AbstractObjectSerializer.class, + "beginConstructorRef", + PRIMITIVE_VOID_TYPE, + readContextRef())); + List bufferedNonConstructorFields = new ArrayList<>(); + int remainingConstructorFields = countConstructorFields(); + Expression bean = null; + if (remainingConstructorFields == 0) { + bean = createCtorBean(expressions, fieldsArray); + } + for (Descriptor descriptor : protocolDescriptors()) { + int index = fieldIndexes.get(descriptor); + walkPath.add(descriptor.getDeclaringClass() + descriptor.getName()); + if (constructorFieldMask[index]) { + expressions.add(deserializeToFieldsArray(fieldsArray, buffer, descriptor, true)); + remainingConstructorFields--; + if (remainingConstructorFields == 0) { + bean = createCtorBean(expressions, fieldsArray); + addBufferedFieldSetters(expressions, bean, fieldsArray, bufferedNonConstructorFields); + } + } else if (bean == null) { + expressions.add(deserializeToFieldsArray(fieldsArray, buffer, descriptor, false)); + bufferedNonConstructorFields.add(descriptor); + } else { + expressions.add(deserializeToBean(bean, buffer, descriptor)); + } + walkPath.removeLast(); + } + expressions.add( + new StaticInvoke( + AbstractObjectSerializer.class, + "endConstructorRef", + PRIMITIVE_VOID_TYPE, + readContextRef())); + expressions.add(new Expression.Return(bean)); + return expressions; + } + + private int countConstructorFields() { + int count = 0; + for (boolean constructorField : constructorFieldMask) { + if (constructorField) { + count++; + } + } + return count; + } + + private List protocolDescriptors() { + List descriptors = new ArrayList<>(); + addDescriptors(descriptors, objectCodecOptimizer.primitiveGroups); + addDescriptors(descriptors, objectCodecOptimizer.boxedReadGroups); + addDescriptors(descriptors, objectCodecOptimizer.nonPrimitiveReadGroups); + return descriptors; + } + + private void addDescriptors(List descriptors, List> groups) { + for (List group : groups) { + descriptors.addAll(group); + } + } + + private Expression createCtorBean(ListExpression expressions, FieldsArray fieldsArray) { + Expression bean = createConstructorObject(fieldsArray); + expressions.add( + new StaticInvoke( + AbstractObjectSerializer.class, + "checkNoUnresolvedReadRef", + PRIMITIVE_VOID_TYPE, + readContextRef(), + staticBeanClassExpr())); + expressions.add(bean); + expressions.add( + new StaticInvoke( + AbstractObjectSerializer.class, + "referenceConstructorRef", + PRIMITIVE_VOID_TYPE, + readContextRef(), + bean)); + postCreateConstructorObject(expressions, bean); + return bean; + } + + private Expression deserializeToFieldsArray( + FieldsArray fieldsArray, Reference buffer, Descriptor descriptor, boolean constructorField) { + TypeRef castTypeRef = + hasCompatibleCollectionArrayRead(descriptor) + ? compatibleReadTargetTypeRef(descriptor) + : descriptor.getTypeRef(); + return deserializeField( + buffer, + descriptor, + expr -> { + Expression value = + constructorField ? tryInlineCast(expr, castTypeRef) : new Cast(expr, OBJECT_TYPE); + value = + new StaticInvoke( + AbstractObjectSerializer.class, + constructorField ? "ctorFieldValue" : "bufferFieldValue", + OBJECT_TYPE, + readContextRef(), + value, + staticBeanClassExpr()); + return setFieldValue(fieldsArray, descriptor, value); + }); + } + + private Expression deserializeToBean(Expression bean, Reference buffer, Descriptor descriptor) { + TypeRef castTypeRef = + hasCompatibleCollectionArrayRead(descriptor) + ? compatibleReadTargetTypeRef(descriptor) + : descriptor.getTypeRef(); + return deserializeField( + buffer, descriptor, expr -> setFieldValue(bean, descriptor, tryInlineCast(expr, castTypeRef))); + } + + protected void postCreateConstructorObject(ListExpression expressions, Expression bean) {} + + protected void deserializeReadGroup( + List> readGroups, + int numGroups, + ListExpression expressions, + Expression bean, + Reference buffer) { + for (List group : readGroups) { + if (group.isEmpty()) { + continue; + } + boolean inline = hasFewFields() || (group.size() == 1 && numGroups < 10); + expressions.add(deserializeGroup(group, bean, buffer, inline)); + } + } + + protected Expression buildComponentsArray() { + return new StaticInvoke( + UnsafeOps.class, "copyObjectArray", OBJECT_ARRAY_TYPE, recordComponentDefaultValues); + } + + protected Expression createRecord(SortedMap recordComponents) { + Expression[] params = recordComponents.values().toArray(new Expression[0]); + return new NewInstance(beanType, params); + } + + protected Expression createConstructorObject(FieldsArray fieldValues) { + Expression[] params = new Expression[constructorFieldIndexes.length]; + Expression[] directParams = new Expression[constructorFieldIndexes.length]; + for (int i = 0; i < constructorFieldIndexes.length; i++) { + int index = constructorFieldIndexes[i]; + if (index < 0) { + params[i] = defaultConstructorValue(i); + } else { + params[i] = fieldValue(fieldValues, index); + } + directParams[i] = tryInlineCast(params[i], TypeRef.of(constructorFieldTypes[i])); + } + ObjectCreator objectCreator = ObjectCreators.getObjectCreator(beanClass); + if (objectCreator.isOnlyPublicConstructor() + && sourcePublicAccessible(beanClass) + && constructorParamsAccessible()) { + return new NewInstance(beanType, directParams); + } + Expression args = new Expression.NewArray(OBJECT_ARRAY_TYPE, params); + Expression newInstance = + new Invoke(getObjectCreator(beanClass), "newInstanceWithArguments", OBJECT_TYPE, args); + return sourcePublicAccessible(beanClass) ? new Cast(newInstance, beanType) : newInstance; + } + + protected Expression defaultConstructorValue(int constructorParameterIndex) { + return new StaticInvoke( + AbstractObjectSerializer.class, + "defaultConstructorValue", + OBJECT_TYPE, + staticClassFieldExpr( + constructorFieldTypes[constructorParameterIndex], + "constructorFieldClass" + constructorParameterIndex + "_")); + } + + private boolean constructorParamsAccessible() { + for (Class constructorFieldType : constructorFieldTypes) { + if (!sourcePublicAccessible(constructorFieldType)) { + return false; + } + } + return true; + } + + private void addNonConstructorFieldSetters( + ListExpression expressions, Expression bean, FieldsArray fieldValues) { + for (Descriptor descriptor : objectCodecOptimizer.descriptorGrouper.getSortedDescriptors()) { + int index = fieldIndexes.get(descriptor); + if (constructorFieldMask[index]) { + continue; + } + TypeRef castTypeRef = + hasCompatibleCollectionArrayRead(descriptor) + ? compatibleReadTargetTypeRef(descriptor) + : descriptor.getTypeRef(); + Expression value = + new StaticInvoke( + AbstractObjectSerializer.class, + "resolveBufferedValue", + OBJECT_TYPE, + fieldValue(fieldValues, index), + bean); + value = tryInlineCast(value, castTypeRef); + expressions.add(setFieldValue(bean, descriptor, value)); + } + } + + private void addBufferedFieldSetters( + ListExpression expressions, + Expression bean, + FieldsArray fieldValues, + List descriptors) { + for (Descriptor descriptor : descriptors) { + int index = fieldIndexes.get(descriptor); + TypeRef castTypeRef = + hasCompatibleCollectionArrayRead(descriptor) + ? compatibleReadTargetTypeRef(descriptor) + : descriptor.getTypeRef(); + Expression value = + new StaticInvoke( + AbstractObjectSerializer.class, + "resolveBufferedValue", + OBJECT_TYPE, + fieldValue(fieldValues, index), + bean); + value = tryInlineCast(value, castTypeRef); + expressions.add(setFieldValue(bean, descriptor, value)); + } + } + + private Expression fieldValue(Expression fieldValues, int index) { + return new StaticInvoke( + AbstractObjectSerializer.class, + "fieldValue", + OBJECT_TYPE, + fieldValues, + Literal.ofInt(index)); + } + + private class FieldsCollector extends Expression.AbstractExpression { + private final TreeMap recordValuesMap = new TreeMap<>(); + + protected FieldsCollector() { + super(new Expression[0]); + } + + @Override + public TypeRef type() { + return beanType; + } + + @Override + public Code.ExprCode doGenCode(CodegenContext ctx) { + return new Code.ExprCode(FalseLiteral, Code.variable(getRawType(beanType), "null")); + } + } + + protected class FieldsArray extends Expression.AbstractExpression { + private final int size; + private final String name; + + protected FieldsArray(int size) { + super(new Expression[0]); + this.size = size; + name = ctx.newName("fieldValues"); + } + + @Override + public TypeRef type() { + return OBJECT_ARRAY_TYPE; + } + + @Override + public Code.ExprCode doGenCode(CodegenContext ctx) { + String code = ctx.type(Object[].class) + " " + name + " = new Object[" + size + "];"; + return new Code.ExprCode(code, FalseLiteral, Code.variable(Object[].class, name)); + } + + int fieldIndex(Descriptor descriptor) { + return fieldIndexes.get(descriptor); + } + } + + @Override + protected Expression setFieldValue(Expression bean, Descriptor d, Expression value) { + if (bean instanceof FieldsArray) { + return new Expression.AssignArrayElem( + bean, value, Literal.ofInt(((FieldsArray) bean).fieldIndex(d))); + } + if (isRecord) { + if (recordCtrAccessible) { + if (value instanceof Inlineable) { + ((Inlineable) value).inline(false); + } + int index = recordReversedMapping.get(d.getName()); + FieldsCollector collector = (FieldsCollector) bean; + collector.recordValuesMap.put(index, value); + return value; + } else { + int index = recordReversedMapping.get(d.getName()); + return new Expression.AssignArrayElem(bean, value, Literal.ofInt(index)); + } + } + return super.setFieldValue(bean, d, value); + } + + protected Expression deserializeGroup( + List group, Expression bean, Expression buffer, boolean inline) { + if (isRecord) { + return deserializeGroupForRecord(group, bean, buffer); + } + SerializableSupplier exprSupplier = + () -> { + ListExpression groupExpressions = new ListExpression(); + // use Reference to cut-off expr dependency. + for (Descriptor d : group) { + ExpressionVisitor.ExprHolder exprHolder = ExpressionVisitor.ExprHolder.of("bean", bean); + walkPath.add(d.getDeclaringClass() + d.getName()); + TypeRef castTypeRef = + hasCompatibleCollectionArrayRead(d) + ? compatibleReadTargetTypeRef(d) + : d.getTypeRef(); + Expression action = + deserializeField( + buffer, + d, + // `bean` will be replaced by `Reference` to cut-off expr + // dependency. + expr -> + setFieldValue(exprHolder.get("bean"), d, tryInlineCast(expr, castTypeRef))); + walkPath.removeLast(); + if (needsGeneratedReadFieldMethod(d)) { + action = + objectCodecOptimizer.invokeGenerated( + readCutPoints(bean, buffer), action, "readField"); + } + groupExpressions.add(action); + } + return groupExpressions; + }; + if (inline) { + return exprSupplier.get(); + } else { + return objectCodecOptimizer.invokeGenerated( + readCutPoints(bean, buffer), exprSupplier.get(), "readFields"); + } + } + + private boolean needsGeneratedReadFieldMethod(Descriptor descriptor) { + return !hasFewFields() + && !isMonomorphic(descriptor) + && !useCollectionSerialization(descriptor) + && !useMapSerialization(descriptor.getTypeRef()); + } + + protected Expression deserializeGroupForRecord( + List group, Expression bean, Expression buffer) { + ListExpression groupExpressions = new ListExpression(); + // use Reference to cut-off expr dependency. + for (Descriptor d : group) { + TypeRef castTypeRef = + hasCompatibleCollectionArrayRead(d) ? compatibleReadTargetTypeRef(d) : d.getTypeRef(); + Expression value = deserializeField(buffer, d, expr -> expr); + Expression action = setFieldValue(bean, d, tryInlineCast(value, castTypeRef)); + groupExpressions.add(action); + } + return groupExpressions; + } + + private Expression checkClassVersion(Expression buffer) { + return new StaticInvoke( + ObjectSerializer.class, + "checkClassVersion", + PRIMITIVE_VOID_TYPE, + false, + beanClassExpr(), + inlineInvoke(buffer, readIntFunc(), PRIMITIVE_INT_TYPE), + Objects.requireNonNull(classVersionHash)); + } + + /** + * Return a list of expressions that deserialize all primitive fields. This can reduce unnecessary + * check call and increment readerIndex in writeXXX. + */ + protected List deserializePrimitives( + Expression bean, Expression buffer, List> primitiveGroups) { + int totalSize = getTotalSizeOfPrimitives(primitiveGroups); + if (totalSize == 0) { + return new ArrayList<>(); + } + if (config.compressInt() || config.compressLong()) { + return deserializeCompressedPrimitives(bean, buffer, primitiveGroups); + } else { + return deserializeUnCompressedPrimitives(bean, buffer, primitiveGroups, totalSize); + } + } + + private List deserializeUnCompressedPrimitives( + Expression bean, Expression buffer, List> primitiveGroups, int totalSize) { + List expressions = new ArrayList<>(); + int numPrimitiveFields = getNumPrimitiveFields(primitiveGroups); + Literal totalSizeLiteral = Literal.ofInt(totalSize); + // After this check, following read can be totally unsafe without checks + expressions.add(new Invoke(buffer, "checkReadableBytes", totalSizeLiteral)); + Expression readerIndex = + new Invoke(buffer, "readerIndex", "readerIndex", PRIMITIVE_INT_TYPE); + expressions.add(readerIndex); + int acc = 0; + for (List group : primitiveGroups) { + ListExpression groupExpressions = new ListExpression(); + for (Descriptor descriptor : group) { + int dispatchId = getNumericDescriptorDispatchId(descriptor); + Expression fieldValue; + if (dispatchId == DispatchId.BOOL) { + fieldValue = bufferGetBoolean(buffer, getBufferIndex(readerIndex, acc)); + acc += 1; + } else if (dispatchId == DispatchId.INT8) { + fieldValue = bufferGetByte(buffer, getBufferIndex(readerIndex, acc)); + acc += 1; + } else if (dispatchId == DispatchId.UINT8) { + fieldValue = + new StaticInvoke( + Byte.class, + "toUnsignedInt", + descriptor.getTypeRef(), + bufferGetByte(buffer, getBufferIndex(readerIndex, acc))); + acc += 1; + } else if (dispatchId == DispatchId.CHAR) { + fieldValue = bufferGetChar(buffer, getBufferIndex(readerIndex, acc)); + acc += 2; + } else if (dispatchId == DispatchId.INT16) { + fieldValue = bufferGetInt16(buffer, getBufferIndex(readerIndex, acc)); + acc += 2; + } else if (dispatchId == DispatchId.UINT16) { + fieldValue = + new StaticInvoke( + Short.class, + "toUnsignedInt", + descriptor.getTypeRef(), + bufferGetInt16(buffer, getBufferIndex(readerIndex, acc))); + acc += 2; + } else if (dispatchId == DispatchId.FLOAT16) { + fieldValue = + new StaticInvoke( + Float16.class, + "fromBits", + TypeRef.of(Float16.class), + bufferGetInt16(buffer, getBufferIndex(readerIndex, acc))); + acc += 2; + } else if (dispatchId == DispatchId.BFLOAT16) { + fieldValue = + new StaticInvoke( + BFloat16.class, + "fromBits", + TypeRef.of(BFloat16.class), + bufferGetInt16(buffer, getBufferIndex(readerIndex, acc))); + acc += 2; + } else if (dispatchId == DispatchId.INT32) { + fieldValue = bufferGetInt32(buffer, getBufferIndex(readerIndex, acc)); + acc += 4; + } else if (dispatchId == DispatchId.UINT32) { + fieldValue = + new StaticInvoke( + Integer.class, + "toUnsignedLong", + descriptor.getTypeRef(), + bufferGetInt32(buffer, getBufferIndex(readerIndex, acc))); + acc += 4; + } else if (dispatchId == DispatchId.INT64 || dispatchId == DispatchId.UINT64) { + fieldValue = bufferGetInt64(buffer, getBufferIndex(readerIndex, acc)); + acc += 8; + } else if (dispatchId == DispatchId.FLOAT32) { + fieldValue = bufferGetFloat32(buffer, getBufferIndex(readerIndex, acc)); + acc += 4; + } else if (dispatchId == DispatchId.FLOAT64) { + fieldValue = bufferGetFloat64(buffer, getBufferIndex(readerIndex, acc)); + acc += 8; + } else { + throw new IllegalStateException("Unsupported dispatchId: " + dispatchId); + } + // `bean` will be replaced by `Reference` to cut-off expr dependency. + groupExpressions.add(setFieldValue(bean, descriptor, fieldValue)); + } + if (hasFewFields() || numPrimitiveFields < 4 || isRecord) { + expressions.add(groupExpressions); + } else { + expressions.add( + objectCodecOptimizer.invokeGenerated( + ofHashSet(bean, buffer, readerIndex), groupExpressions, "readFields")); + } + } + Expression increaseReaderIndex = + new Invoke( + buffer, "increaseReaderIndex", new Literal(totalSizeLiteral, PRIMITIVE_INT_TYPE)); + expressions.add(increaseReaderIndex); + return expressions; + } + + private List deserializeCompressedPrimitives( + Expression bean, Expression buffer, List> primitiveGroups) { + List expressions = new ArrayList<>(); + int numPrimitiveFields = getNumPrimitiveFields(primitiveGroups); + for (List group : primitiveGroups) { + // After this check, following read can be totally unsafe without checks. + // checkReadableBytes first, `fillBuffer` may create a new heap buffer. + ReplaceStub checkReadableBytesStub = new ReplaceStub(); + expressions.add(checkReadableBytesStub); + Expression readerIndex = + new Invoke(buffer, "readerIndex", "readerIndex", PRIMITIVE_INT_TYPE); + expressions.add(readerIndex); + ListExpression groupExpressions = new ListExpression(); + int acc = 0; + boolean compressStarted = false; + for (Descriptor descriptor : group) { + int dispatchId = getNumericDescriptorDispatchId(descriptor); + Expression fieldValue; + if (dispatchId == DispatchId.BOOL) { + fieldValue = bufferGetBoolean(buffer, getBufferIndex(readerIndex, acc)); + acc += 1; + } else if (dispatchId == DispatchId.INT8) { + fieldValue = bufferGetByte(buffer, getBufferIndex(readerIndex, acc)); + acc += 1; + } else if (dispatchId == DispatchId.UINT8) { + fieldValue = + new StaticInvoke( + Byte.class, + "toUnsignedInt", + descriptor.getTypeRef(), + bufferGetByte(buffer, getBufferIndex(readerIndex, acc))); + acc += 1; + } else if (dispatchId == DispatchId.CHAR) { + fieldValue = bufferGetChar(buffer, getBufferIndex(readerIndex, acc)); + acc += 2; + } else if (dispatchId == DispatchId.INT16) { + fieldValue = bufferGetInt16(buffer, getBufferIndex(readerIndex, acc)); + acc += 2; + } else if (dispatchId == DispatchId.UINT16) { + fieldValue = + new StaticInvoke( + Short.class, + "toUnsignedInt", + descriptor.getTypeRef(), + bufferGetInt16(buffer, getBufferIndex(readerIndex, acc))); + acc += 2; + } else if (dispatchId == DispatchId.FLOAT16) { + fieldValue = + new StaticInvoke( + Float16.class, + "fromBits", + TypeRef.of(Float16.class), + bufferGetInt16(buffer, getBufferIndex(readerIndex, acc))); + acc += 2; + } else if (dispatchId == DispatchId.BFLOAT16) { + fieldValue = + new StaticInvoke( + BFloat16.class, + "fromBits", + TypeRef.of(BFloat16.class), + bufferGetInt16(buffer, getBufferIndex(readerIndex, acc))); + acc += 2; + } else if (dispatchId == DispatchId.FLOAT32) { + fieldValue = bufferGetFloat32(buffer, getBufferIndex(readerIndex, acc)); + acc += 4; + } else if (dispatchId == DispatchId.FLOAT64) { + fieldValue = bufferGetFloat64(buffer, getBufferIndex(readerIndex, acc)); + acc += 8; + } else if (dispatchId == DispatchId.INT32) { + fieldValue = bufferGetInt32(buffer, getBufferIndex(readerIndex, acc)); + acc += 4; + } else if (dispatchId == DispatchId.UINT32) { + fieldValue = + new StaticInvoke( + Integer.class, + "toUnsignedLong", + descriptor.getTypeRef(), + bufferGetInt32(buffer, getBufferIndex(readerIndex, acc))); + acc += 4; + } else if (dispatchId == DispatchId.INT64 || dispatchId == DispatchId.UINT64) { + fieldValue = bufferGetInt64(buffer, getBufferIndex(readerIndex, acc)); + acc += 8; + } else if (dispatchId == DispatchId.VARINT32) { + if (!compressStarted) { + compressStarted = true; + addIncReaderIndexExpr(groupExpressions, buffer, acc); + } + fieldValue = readVarInt32(buffer); + } else if (dispatchId == DispatchId.VAR_UINT32) { + if (!compressStarted) { + compressStarted = true; + addIncReaderIndexExpr(groupExpressions, buffer, acc); + } + fieldValue = + new StaticInvoke( + Integer.class, + "toUnsignedLong", + descriptor.getTypeRef(), + new Invoke(buffer, "readVarUInt32", PRIMITIVE_INT_TYPE)); + } else if (dispatchId == DispatchId.VARINT64) { + if (!compressStarted) { + compressStarted = true; + addIncReaderIndexExpr(groupExpressions, buffer, acc); + } + fieldValue = new Invoke(buffer, "readVarInt64", PRIMITIVE_LONG_TYPE); + } else if (dispatchId == DispatchId.TAGGED_INT64) { + if (!compressStarted) { + compressStarted = true; + addIncReaderIndexExpr(groupExpressions, buffer, acc); + } + fieldValue = new Invoke(buffer, "readTaggedInt64", PRIMITIVE_LONG_TYPE); + } else if (dispatchId == DispatchId.VAR_UINT64) { + if (!compressStarted) { + compressStarted = true; + addIncReaderIndexExpr(groupExpressions, buffer, acc); + } + fieldValue = new Invoke(buffer, "readVarUInt64", PRIMITIVE_LONG_TYPE); + } else if (dispatchId == DispatchId.TAGGED_UINT64) { + if (!compressStarted) { + compressStarted = true; + addIncReaderIndexExpr(groupExpressions, buffer, acc); + } + fieldValue = new Invoke(buffer, "readTaggedUInt64", PRIMITIVE_LONG_TYPE); + } else { + throw new IllegalStateException("Unsupported dispatchId: " + dispatchId); + } + // `bean` will be replaced by `Reference` to cut-off expr dependency. + groupExpressions.add(setFieldValue(bean, descriptor, fieldValue)); + } + if (acc != 0) { + checkReadableBytesStub.setTargetObject( + new Invoke(buffer, "checkReadableBytes", Literal.ofInt(acc))); + } + if (!compressStarted) { + addIncReaderIndexExpr(groupExpressions, buffer, acc); + } + if (hasFewFields() || numPrimitiveFields < 4 || isRecord) { + expressions.add(groupExpressions); + } else { + expressions.add( + objectCodecOptimizer.invokeGenerated( + ofHashSet(bean, buffer, readerIndex), groupExpressions, "readFields")); + } + } + return expressions; + } + + private void addIncReaderIndexExpr(ListExpression expressions, Expression buffer, int diff) { + if (diff != 0) { + expressions.add(new Invoke(buffer, "increaseReaderIndex", Literal.ofInt(diff))); + } + } + + private Expression getReaderAddress(Expression readerPos, long acc) { + if (acc == 0) { + return readerPos; + } + return add(readerPos, new Literal(acc, PRIMITIVE_LONG_TYPE)); + } + + private Expression getBufferIndex(Expression index, int acc) { + if (acc == 0) { + return index; + } + return add(index, Literal.ofInt(acc)); + } +} diff --git a/java/fory-core/src/main/java25/org/apache/fory/memory/MemoryBuffer.java b/java/fory-core/src/main/java25/org/apache/fory/memory/MemoryBuffer.java new file mode 100644 index 0000000000..82203215cf --- /dev/null +++ b/java/fory-core/src/main/java25/org/apache/fory/memory/MemoryBuffer.java @@ -0,0 +1,4521 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fory.memory; + +import static org.apache.fory.util.Preconditions.checkArgument; +import static org.apache.fory.util.Preconditions.checkNotNull; + +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; +import java.lang.invoke.MethodHandles; +import java.lang.invoke.VarHandle; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.Arrays; +import org.apache.fory.annotation.CodegenInvoke; +import org.apache.fory.io.AbstractStreamReader; +import org.apache.fory.io.ForyStreamReader; +import org.apache.fory.platform.AndroidSupport; +import org.apache.fory.platform.UnsafeOps; + +/** + * A class for operations on memory managed by Fory. The buffer may be backed by heap memory (byte + * array) or by off-heap memory. Note that the buffer can auto grow on write operations and change + * into a heap buffer when growing. + * + *

This is a byte buffer similar class with more features: + * + *

    + *
  • read/write data into a chunk of direct memory. + *
  • additional binary compare, swap, and copy methods. + *
  • little-endian access. + *
  • independent read/write index. + *
  • varint int/long encoding. + *
  • aligned int/long encoding. + *
+ * + *

Note that this class is designed to final so that all the methods in this class can be inlined + * by the just-in-time compiler. + * + *

TODO(chaokunyang) Let grow/readerIndex/writerIndex handled in this class and Make immutable + * part as separate class, and use composition in this class. In this way, all fields can be final + * and access will be much faster. + * + *

Warning: The instance of this class should not be hold on graalvm build time, the heap unsafe + * offset are not correct in runtime since graalvm will change array base offset. + * + *

Note(chaokunyang): Buffer operations are very common, and jvm inline and branch elimination is + * not reliable even in c2 compiler, so we try to inline and avoid checks as we can manually. jvm + * jit may stop inline for some reasons: NodeCountInliningCutoff, + * DesiredMethodLimit,MaxRecursiveInlineLevel,FreqInlineSize,MaxInlineSize + */ +public final class MemoryBuffer { + public static final int BUFFER_GROW_STEP_THRESHOLD = 100 * 1024 * 1024; + private static final boolean LITTLE_ENDIAN = NativeByteOrder.IS_LITTLE_ENDIAN; + private static final boolean UNALIGNED = !AndroidSupport.IS_ANDROID && UnsafeOps.unaligned(); + private static final ByteOrder NATIVE_ORDER = ByteOrder.nativeOrder(); + private static final VarHandle BYTE_ARRAY_CHAR = + MethodHandles.byteArrayViewVarHandle(char[].class, NATIVE_ORDER); + private static final VarHandle BYTE_ARRAY_SHORT = + MethodHandles.byteArrayViewVarHandle(short[].class, NATIVE_ORDER); + private static final VarHandle BYTE_ARRAY_INT = + MethodHandles.byteArrayViewVarHandle(int[].class, NATIVE_ORDER); + private static final VarHandle BYTE_ARRAY_LONG = + MethodHandles.byteArrayViewVarHandle(long[].class, NATIVE_ORDER); + private static final VarHandle BYTE_BUFFER_CHAR = + MethodHandles.byteBufferViewVarHandle(char[].class, NATIVE_ORDER); + private static final VarHandle BYTE_BUFFER_SHORT = + MethodHandles.byteBufferViewVarHandle(short[].class, NATIVE_ORDER); + private static final VarHandle BYTE_BUFFER_INT = + MethodHandles.byteBufferViewVarHandle(int[].class, NATIVE_ORDER); + private static final VarHandle BYTE_BUFFER_LONG = + MethodHandles.byteBufferViewVarHandle(long[].class, NATIVE_ORDER); + private static final ValueLayout.OfChar NATIVE_CHAR = + ValueLayout.JAVA_CHAR_UNALIGNED.withOrder(NATIVE_ORDER); + private static final ValueLayout.OfShort NATIVE_SHORT = + ValueLayout.JAVA_SHORT_UNALIGNED.withOrder(NATIVE_ORDER); + private static final ValueLayout.OfInt NATIVE_INT = + ValueLayout.JAVA_INT_UNALIGNED.withOrder(NATIVE_ORDER); + private static final ValueLayout.OfLong NATIVE_LONG = + ValueLayout.JAVA_LONG_UNALIGNED.withOrder(NATIVE_ORDER); + // Global allocator instance that can be customized + private static volatile MemoryAllocator globalAllocator = new DefaultMemoryAllocator(); + + // If the data in on the heap, `heapMemory` will be non-null, and its' the object relative to + // which we access the memory. + // If we have this buffer, we must never void this reference, or the memory buffer will point + // to undefined addresses outside the heap and may in out-of-order execution cases cause + // buffer faults. + byte[] heapMemory; + int heapOffset; + // If the data is off the heap, `offHeapBuffer` will be non-null, and it's the direct byte buffer + // that allocated on the off-heap memory. + // This memory buffer holds a reference to that buffer, so as long as this memory buffer lives, + // the memory will not be released. + ByteBuffer offHeapBuffer; + ByteBuffer nativeOffHeapBuffer; + MemorySegment offHeapSegment; + // The readable/writeable range is [address, addressLimit). + // If the data in on the heap, this is the relative offset to the `heapMemory` byte array. + // If the data is off the heap, this is the logical byte index into `offHeapBuffer`. + long address; + // The address one byte after the last addressable byte, i.e. `address + size` while the + // buffer is not disposed. + long addressLimit; + // The size in bytes of the memory buffer. + int size; + int readerIndex; + int writerIndex; + final ForyStreamReader streamReader; + private final MemoryAccess memoryAccess = new MemoryAccess(this); + + // Android branches in this class are intentional method-boundary exits. + // Do not delete them or fold them into the JVM Unsafe path: each branch must make exactly one + // MemoryOps call, while MemoryOps owns Android heap index math and reader/writer updates. + + /** + * Creates a new memory buffer that represents the memory of the byte array. + * + * @param buffer The byte array whose memory is represented by this memory buffer. + * @param offset The offset of the sub array to be used; must be non-negative and no larger than + * array.length. + * @param length buffer size + */ + private MemoryBuffer(byte[] buffer, int offset, int length) { + this(buffer, offset, length, null); + } + + /** + * Creates a new memory buffer that represents the memory of the byte array. + * + * @param buffer The byte array whose memory is represented by this memory buffer. + * @param offset The offset of the sub array to be used; must be non-negative and no larger than + * array.length. + * @param length buffer size + * @param streamReader a reader for reading from a stream. + */ + private MemoryBuffer(byte[] buffer, int offset, int length, ForyStreamReader streamReader) { + checkArgument(offset >= 0 && length >= 0); + if (offset + length > buffer.length) { + throw new IllegalArgumentException( + String.format("%d exceeds buffer size %d", offset + length, buffer.length)); + } + initHeapBuffer(buffer, offset, length); + if (streamReader != null) { + this.streamReader = streamReader; + } else { + this.streamReader = new BoundChecker(); + } + } + + /** + * Creates a new memory buffer that represents the native memory at the absolute address given by + * the pointer. + * + * @param offHeapAddress The address of the memory represented by this memory buffer. + * @param size The size of this memory buffer. + * @param offHeapBuffer The byte buffer whose memory is represented by this memory buffer which + * may be null if the memory is not allocated by `DirectByteBuffer`. Hold this buffer to avoid + * the memory being released. + */ + private MemoryBuffer(long offHeapAddress, int size, ByteBuffer offHeapBuffer) { + this(offHeapAddress, size, offHeapBuffer, null); + } + + /** + * Creates a new memory buffer that represents the native memory at the absolute address given by + * the pointer. + * + * @param offHeapAddress The address of the memory represented by this memory buffer. + * @param size The size of this memory buffer. + * @param offHeapBuffer The byte buffer whose memory is represented by this memory buffer which + * may be null if the memory is not allocated by `DirectByteBuffer`. Hold this buffer to avoid + * the memory being released. + * @param streamReader a reader for reading from a stream. + */ + private MemoryBuffer( + long offHeapAddress, int size, ByteBuffer offHeapBuffer, ForyStreamReader streamReader) { + initOffHeapBuffer(offHeapAddress, size, offHeapBuffer); + if (streamReader != null) { + this.streamReader = streamReader; + } else { + this.streamReader = new BoundChecker(); + } + } + + private void initOffHeapBuffer(long offHeapAddress, int size, ByteBuffer offHeapBuffer) { + checkArgument(offHeapAddress >= 0 && size >= 0); + checkNotNull(offHeapBuffer, "JDK25 MemoryBuffer requires a ByteBuffer owner for off-heap data"); + checkArgument(offHeapBuffer.isDirect(), "Only direct ByteBuffers can back off-heap MemoryBuffer"); + this.offHeapBuffer = offHeapBuffer; + this.nativeOffHeapBuffer = offHeapBuffer.duplicate().order(NATIVE_ORDER); + ByteBuffer segmentBuffer = offHeapBuffer.duplicate(); + ByteBufferUtil.position(segmentBuffer, 0); + this.offHeapSegment = MemorySegment.ofBuffer(segmentBuffer); + this.heapMemory = null; + this.address = offHeapAddress; + this.addressLimit = this.address + size; + this.size = size; + } + + private static long getAddress(ByteBuffer buffer) { + checkNotNull(buffer, "buffer is null"); + checkArgument(buffer.isDirect(), "Can't get address of a non-direct ByteBuffer."); + throw new UnsupportedOperationException("Raw direct-buffer addresses are not exposed on JDK25"); + } + + public void initByteBuffer(ByteBuffer buffer, int size) { + if (buffer.isDirect()) { + if (AndroidSupport.IS_ANDROID) { + MemoryOps.throwDirectByteBufferUnsupported(); + } else { + initOffHeapBuffer(0, size, buffer); + } + } else if (buffer.hasArray()) { + initHeapBuffer(buffer.array(), buffer.arrayOffset(), size); + } else { + throw new IllegalArgumentException("ByteBuffer must be direct or expose an array"); + } + } + + private class BoundChecker extends AbstractStreamReader { + @Override + public int fillBuffer(int minFillSize) { + throw new IndexOutOfBoundsException( + String.format( + "readerIndex(%d) + length(%d) exceeds size(%d): %s", + readerIndex, minFillSize, size, this)); + } + + @Override + public void readTo(byte[] dst, int dstIndex, int length) { + throwIndexOOBExceptionForRead(length); + } + + @Override + public void readToByteBuffer(ByteBuffer dst, int length) { + throwIndexOOBExceptionForRead(length); + } + + @Override + public int readToByteBuffer(ByteBuffer dst) { + throwIndexOOBExceptionForRead(dst.remaining()); + return 0; + } + + @Override + public MemoryBuffer getBuffer() { + return MemoryBuffer.this; + } + } + + public void initHeapBuffer(byte[] buffer, int offset, int length) { + if (AndroidSupport.IS_ANDROID) { + MemoryOps.initHeapBuffer(this, buffer, offset, length); + } else { + if (buffer == null) { + throw new NullPointerException("buffer"); + } + this.heapMemory = buffer; + this.heapOffset = offset; + this.offHeapBuffer = null; + this.nativeOffHeapBuffer = null; + this.offHeapSegment = null; + // Versioned UnsafeOps array base offsets are zero on JDK25; address is a logical byte index. + final long startPos = UnsafeOps.BYTE_ARRAY_OFFSET + offset; + this.address = startPos; + this.size = length; + this.addressLimit = startPos + length; + } + } + + private byte loadByte(long pos) { + byte[] heap = heapMemory; + if (heap != null) { + return heap[(int) pos]; + } + return nativeOffHeapBuffer.get((int) pos); + } + + private void storeByte(long pos, byte value) { + byte[] heap = heapMemory; + if (heap != null) { + heap[(int) pos] = value; + } else { + nativeOffHeapBuffer.put((int) pos, value); + } + } + + private char loadChar(long pos) { + byte[] heap = heapMemory; + if (heap != null) { + return (char) BYTE_ARRAY_CHAR.get(heap, (int) pos); + } + return (char) BYTE_BUFFER_CHAR.get(nativeOffHeapBuffer, (int) pos); + } + + private void storeChar(long pos, char value) { + byte[] heap = heapMemory; + if (heap != null) { + BYTE_ARRAY_CHAR.set(heap, (int) pos, value); + } else { + BYTE_BUFFER_CHAR.set(nativeOffHeapBuffer, (int) pos, value); + } + } + + private short loadShort(long pos) { + byte[] heap = heapMemory; + if (heap != null) { + return (short) BYTE_ARRAY_SHORT.get(heap, (int) pos); + } + return (short) BYTE_BUFFER_SHORT.get(nativeOffHeapBuffer, (int) pos); + } + + private void storeShort(long pos, short value) { + byte[] heap = heapMemory; + if (heap != null) { + BYTE_ARRAY_SHORT.set(heap, (int) pos, value); + } else { + BYTE_BUFFER_SHORT.set(nativeOffHeapBuffer, (int) pos, value); + } + } + + private int loadInt(long pos) { + byte[] heap = heapMemory; + if (heap != null) { + return (int) BYTE_ARRAY_INT.get(heap, (int) pos); + } + return (int) BYTE_BUFFER_INT.get(nativeOffHeapBuffer, (int) pos); + } + + private void storeInt(long pos, int value) { + byte[] heap = heapMemory; + if (heap != null) { + BYTE_ARRAY_INT.set(heap, (int) pos, value); + } else { + BYTE_BUFFER_INT.set(nativeOffHeapBuffer, (int) pos, value); + } + } + + private long loadLong(long pos) { + byte[] heap = heapMemory; + if (heap != null) { + return (long) BYTE_ARRAY_LONG.get(heap, (int) pos); + } + return (long) BYTE_BUFFER_LONG.get(nativeOffHeapBuffer, (int) pos); + } + + private void storeLong(long pos, long value) { + byte[] heap = heapMemory; + if (heap != null) { + BYTE_ARRAY_LONG.set(heap, (int) pos, value); + } else { + BYTE_BUFFER_LONG.set(nativeOffHeapBuffer, (int) pos, value); + } + } + + private static int putVarUInt32Heap(byte[] heap, int index, int value) { + if (value >>> 7 == 0) { + heap[index] = (byte) value; + return 1; + } + heap[index++] = (byte) ((value & 0x7F) | 0x80); + if (value >>> 14 == 0) { + heap[index] = (byte) (value >>> 7); + return 2; + } + heap[index++] = (byte) ((value >>> 7) | 0x80); + if (value >>> 21 == 0) { + heap[index] = (byte) (value >>> 14); + return 3; + } + heap[index++] = (byte) ((value >>> 14) | 0x80); + if (value >>> 28 == 0) { + heap[index] = (byte) (value >>> 21); + return 4; + } + heap[index++] = (byte) ((value >>> 21) | 0x80); + heap[index] = (byte) (value >>> 28); + return 5; + } + + // ------------------------------------------------------------------------ + // Memory buffer Operations + // ------------------------------------------------------------------------ + + /** + * Gets the size of the memory buffer, in bytes. + * + * @return The size of the memory buffer. + */ + public int size() { + return size; + } + + public void increaseSize(int diff) { + this.addressLimit = address + (size += diff); + } + + /** + * Checks whether this memory buffer is backed by off-heap memory. + * + * @return true, if the memory buffer is backed by off-heap memory, false if it + * is backed by heap memory. + */ + public boolean isOffHeap() { + return heapMemory == null; + } + + /** + * Returns true, if the memory buffer is backed by heap memory and memory buffer can + * write to the whole memory region of underlying byte array. + */ + public boolean isHeapFullyWriteable() { + return heapMemory != null && heapOffset == 0; + } + + /** + * Get the heap byte array object. + * + * @return Return non-null if the memory is on the heap, and return null, if the memory if off the + * heap. + */ + public byte[] getHeapMemory() { + return heapMemory; + } + + /** + * Gets the buffer that owns the memory of this memory buffer. + * + * @return The byte buffer that owns the memory of this memory buffer. + */ + public ByteBuffer getOffHeapBuffer() { + if (offHeapBuffer != null) { + return offHeapBuffer; + } else { + throw new IllegalStateException("Memory buffer does not represent off heap ByteBuffer"); + } + } + + /** + * Returns the byte array of on-heap memory buffers. + * + * @return underlying byte array + * @throws IllegalStateException if the memory buffer does not represent on-heap memory + */ + public byte[] getArray() { + if (heapMemory != null) { + return heapMemory; + } else { + throw new IllegalStateException("Memory buffer does not represent heap memory"); + } + } + + // ------------------------------------------------------------------------ + // Random Access get() and put() methods + // ------------------------------------------------------------------------ + + private void checkPosition(long index, long pos, long length) { + if (BoundsChecking.BOUNDS_CHECKING_ENABLED) { + if (index < 0 || pos > addressLimit - length) { + throwOOBException(); + } + } + } + + public void get(int index, byte[] dst) { + get(index, dst, 0, dst.length); + } + + public void get(int index, byte[] dst, int offset, int length) { + final byte[] heapMemory = this.heapMemory; + if (heapMemory != null) { + // System.arraycopy faster for some jdk than Unsafe. + System.arraycopy(heapMemory, heapOffset + index, dst, offset, length); + } else { + final long pos = address + index; + if ((index + | offset + | length + | (offset + length) + | (dst.length - (offset + length)) + | addressLimit - length - pos) + < 0) { + throwOOBException(); + } + memoryAccess.copyMemory(null, pos, dst, UnsafeOps.BYTE_ARRAY_OFFSET + offset, length); + } + } + + public void get(int offset, ByteBuffer target, int numBytes) { + if ((offset | numBytes | (offset + numBytes)) < 0) { + throwOOBException(); + } + if (target.remaining() < numBytes) { + throwOOBException(); + } + if (target.isReadOnly()) { + throw new IllegalArgumentException("read only buffer"); + } + final int targetPos = target.position(); + if (AndroidSupport.IS_ANDROID) { + MemoryOps.get(this, offset, target, numBytes); + } else if (target.isDirect()) { + final long sourceAddr = address + offset; + if (sourceAddr <= addressLimit - numBytes) { + ByteBuffer duplicate = target.duplicate(); + ByteBufferUtil.position(duplicate, targetPos); + duplicate.put(sliceAsByteBuffer(offset, numBytes)); + } else { + throwOOBException(); + } + } else { + assert target.hasArray(); + get(offset, target.array(), targetPos + target.arrayOffset(), numBytes); + } + if (target.position() == targetPos) { + ByteBufferUtil.position(target, targetPos + numBytes); + } + } + + public void put(int offset, ByteBuffer source, int numBytes) { + final int remaining = source.remaining(); + if ((offset | numBytes | (offset + numBytes) | (remaining - numBytes)) < 0) { + throwOOBException(); + } + final int sourcePos = source.position(); + if (AndroidSupport.IS_ANDROID) { + MemoryOps.put(this, offset, source, numBytes); + } else if (source.isDirect()) { + final long targetAddr = address + offset; + if (targetAddr <= addressLimit - numBytes) { + ByteBuffer duplicate = source.duplicate(); + ByteBufferUtil.position(duplicate, sourcePos); + duplicate.limit(sourcePos + numBytes); + if (heapMemory != null) { + duplicate.get(heapMemory, heapOffset + offset, numBytes); + } else { + sliceAsByteBuffer(offset, numBytes).put(duplicate.slice()); + } + } else { + throwOOBException(); + } + } else { + assert source.hasArray(); + put(offset, source.array(), sourcePos + source.arrayOffset(), numBytes); + } + if (source.position() == sourcePos) { + ByteBufferUtil.position(source, sourcePos + numBytes); + } + } + + public void put(int index, byte[] src) { + put(index, src, 0, src.length); + } + + public void put(int index, byte[] src, int offset, int length) { + final byte[] heapMemory = this.heapMemory; + if (heapMemory != null) { + // System.arraycopy faster for some jdk than Unsafe. + System.arraycopy(src, offset, heapMemory, heapOffset + index, length); + } else { + final long pos = address + index; + // check the byte array offset and length + if ((index + | offset + | length + | (offset + length) + | (src.length - (offset + length)) + | addressLimit - length - pos) + < 0) { + throwOOBException(); + } + final long arrayAddress = UnsafeOps.BYTE_ARRAY_OFFSET + offset; + memoryAccess.copyMemory(src, arrayAddress, null, pos, length); + } + } + + public byte getByte(int index) { + if (AndroidSupport.IS_ANDROID) { + return MemoryOps.getByte(this, index); + } + final long pos = address + index; + checkPosition(index, pos, 1); + return loadByte(pos); + } + + // CHECKSTYLE.OFF:MethodName + public byte _unsafeGetByte(int index) { + // CHECKSTYLE.ON:MethodName + if (AndroidSupport.IS_ANDROID) { + return MemoryOps.unsafeGetByte(this, index); + } + return loadByte(address + index); + } + + public void putByte(int index, int b) { + if (AndroidSupport.IS_ANDROID) { + MemoryOps.putByte(this, index, b); + } else { + final long pos = address + index; + checkPosition(index, pos, 1); + storeByte(pos, (byte) b); + } + } + + public void putByte(int index, byte b) { + if (AndroidSupport.IS_ANDROID) { + MemoryOps.putByte(this, index, b); + } else { + final long pos = address + index; + checkPosition(index, pos, 1); + storeByte(pos, b); + } + } + + // CHECKSTYLE.OFF:MethodName + public void _unsafePutByte(int index, byte b) { + // CHECKSTYLE.ON:MethodName + if (AndroidSupport.IS_ANDROID) { + MemoryOps.unsafePutByte(this, index, b); + } else { + storeByte(address + index, b); + } + } + + public boolean getBoolean(int index) { + if (AndroidSupport.IS_ANDROID) { + return MemoryOps.getBoolean(this, index); + } + final long pos = address + index; + checkPosition(index, pos, 1); + return loadByte(pos) != 0; + } + + // CHECKSTYLE.OFF:MethodName + public boolean _unsafeGetBoolean(int index) { + // CHECKSTYLE.ON:MethodName + if (AndroidSupport.IS_ANDROID) { + return MemoryOps.unsafeGetBoolean(this, index); + } + return loadByte(address + index) != 0; + } + + public void putBoolean(int index, boolean value) { + if (AndroidSupport.IS_ANDROID) { + MemoryOps.putBoolean(this, index, value); + } else { + storeByte(address + index, (value ? (byte) 1 : (byte) 0)); + } + } + + // CHECKSTYLE.OFF:MethodName + public void _unsafePutBoolean(int index, boolean value) { + // CHECKSTYLE.ON:MethodName + if (AndroidSupport.IS_ANDROID) { + MemoryOps.putBoolean(this, index, value); + } else { + storeByte(address + index, (value ? (byte) 1 : (byte) 0)); + } + } + + public char getChar(int index) { + if (AndroidSupport.IS_ANDROID) { + return MemoryOps.getChar(this, index); + } + final long pos = address + index; + checkPosition(index, pos, 2); + char c = loadChar(pos); + return LITTLE_ENDIAN ? c : Character.reverseBytes(c); + } + + // CHECKSTYLE.OFF:MethodName + public char _unsafeGetChar(int index) { + // CHECKSTYLE.ON:MethodName + if (AndroidSupport.IS_ANDROID) { + return MemoryOps.unsafeGetChar(this, index); + } + char c = loadChar(address + index); + return LITTLE_ENDIAN ? c : Character.reverseBytes(c); + } + + public void putChar(int index, char value) { + if (AndroidSupport.IS_ANDROID) { + MemoryOps.putChar(this, index, value); + } else { + final long pos = address + index; + checkPosition(index, pos, 2); + if (!LITTLE_ENDIAN) { + value = Character.reverseBytes(value); + } + storeChar(pos, value); + } + } + + // CHECKSTYLE.OFF:MethodName + public void _unsafePutChar(int index, char value) { + // CHECKSTYLE.ON:MethodName + if (AndroidSupport.IS_ANDROID) { + MemoryOps.unsafePutChar(this, index, value); + } else { + if (!LITTLE_ENDIAN) { + value = Character.reverseBytes(value); + } + storeChar(address + index, value); + } + } + + public short getInt16(int index) { + if (AndroidSupport.IS_ANDROID) { + return MemoryOps.getInt16(this, index); + } + final long pos = address + index; + checkPosition(index, pos, 2); + short v = loadShort(pos); + return LITTLE_ENDIAN ? v : Short.reverseBytes(v); + } + + public void putInt16(int index, short value) { + if (AndroidSupport.IS_ANDROID) { + MemoryOps.putInt16(this, index, value); + } else { + final long pos = address + index; + checkPosition(index, pos, 2); + if (!LITTLE_ENDIAN) { + value = Short.reverseBytes(value); + } + storeShort(pos, value); + } + } + + // CHECKSTYLE.OFF:MethodName + public short _unsafeGetInt16(int index) { + // CHECKSTYLE.ON:MethodName + if (AndroidSupport.IS_ANDROID) { + return MemoryOps.unsafeGetInt16(this, index); + } + short v = loadShort(address + index); + return LITTLE_ENDIAN ? v : Short.reverseBytes(v); + } + + // CHECKSTYLE.OFF:MethodName + public void _unsafePutInt16(int index, short value) { + // CHECKSTYLE.ON:MethodName + if (AndroidSupport.IS_ANDROID) { + MemoryOps.unsafePutInt16(this, index, value); + } else { + if (!LITTLE_ENDIAN) { + value = Short.reverseBytes(value); + } + storeShort(address + index, value); + } + } + + public int getInt32(int index) { + if (AndroidSupport.IS_ANDROID) { + return MemoryOps.getInt32(this, index); + } + final long pos = address + index; + checkPosition(index, pos, 4); + int v = loadInt(pos); + return LITTLE_ENDIAN ? v : Integer.reverseBytes(v); + } + + public void putInt32(int index, int value) { + if (AndroidSupport.IS_ANDROID) { + MemoryOps.putInt32(this, index, value); + } else { + final long pos = address + index; + checkPosition(index, pos, 4); + if (!LITTLE_ENDIAN) { + value = Integer.reverseBytes(value); + } + storeInt(pos, value); + } + } + + // CHECKSTYLE.OFF:MethodName + public int _unsafeGetInt32(int index) { + // CHECKSTYLE.ON:MethodName + if (AndroidSupport.IS_ANDROID) { + return MemoryOps.unsafeGetInt32(this, index); + } + int v = loadInt(address + index); + return LITTLE_ENDIAN ? v : Integer.reverseBytes(v); + } + + // CHECKSTYLE.OFF:MethodName + public void _unsafePutInt32(int index, int value) { + // CHECKSTYLE.ON:MethodName + if (AndroidSupport.IS_ANDROID) { + MemoryOps.unsafePutInt32(this, index, value); + } else { + if (!LITTLE_ENDIAN) { + value = Integer.reverseBytes(value); + } + storeInt(address + index, value); + } + } + + public long getInt64(int index) { + if (AndroidSupport.IS_ANDROID) { + return MemoryOps.getInt64(this, index); + } + final long pos = address + index; + checkPosition(index, pos, 8); + long v = loadLong(pos); + return LITTLE_ENDIAN ? v : Long.reverseBytes(v); + } + + public void putInt64(int index, long value) { + if (AndroidSupport.IS_ANDROID) { + MemoryOps.putInt64(this, index, value); + } else { + final long pos = address + index; + checkPosition(index, pos, 8); + if (!LITTLE_ENDIAN) { + value = Long.reverseBytes(value); + } + storeLong(pos, value); + } + } + + // CHECKSTYLE.OFF:MethodName + public long _unsafeGetInt64(int index) { + // CHECKSTYLE.ON:MethodName + if (AndroidSupport.IS_ANDROID) { + return MemoryOps.unsafeGetInt64(this, index); + } + long v = loadLong(address + index); + return LITTLE_ENDIAN ? v : Long.reverseBytes(v); + } + + // CHECKSTYLE.OFF:MethodName + public void _unsafePutInt64(int index, long value) { + // CHECKSTYLE.ON:MethodName + if (AndroidSupport.IS_ANDROID) { + MemoryOps.unsafePutInt64(this, index, value); + } else { + if (!LITTLE_ENDIAN) { + value = Long.reverseBytes(value); + } + storeLong(address + index, value); + } + } + + public float getFloat32(int index) { + if (AndroidSupport.IS_ANDROID) { + return MemoryOps.getFloat32(this, index); + } + final long pos = address + index; + checkPosition(index, pos, 4); + int v = loadInt(pos); + if (!LITTLE_ENDIAN) { + v = Integer.reverseBytes(v); + } + return Float.intBitsToFloat(v); + } + + public void putFloat32(int index, float value) { + if (AndroidSupport.IS_ANDROID) { + MemoryOps.putFloat32(this, index, value); + } else { + final long pos = address + index; + checkPosition(index, pos, 4); + int v = Float.floatToRawIntBits(value); + if (!LITTLE_ENDIAN) { + v = Integer.reverseBytes(v); + } + storeInt(pos, v); + } + } + + public double getFloat64(int index) { + if (AndroidSupport.IS_ANDROID) { + return MemoryOps.getFloat64(this, index); + } + final long pos = address + index; + checkPosition(index, pos, 8); + long v = loadLong(pos); + if (!LITTLE_ENDIAN) { + v = Long.reverseBytes(v); + } + return Double.longBitsToDouble(v); + } + + public void putFloat64(int index, double value) { + if (AndroidSupport.IS_ANDROID) { + MemoryOps.putFloat64(this, index, value); + } else { + final long pos = address + index; + checkPosition(index, pos, 8); + long v = Double.doubleToRawLongBits(value); + if (!LITTLE_ENDIAN) { + v = Long.reverseBytes(v); + } + storeLong(pos, v); + } + } + + // Check should be done outside to avoid this method got into the critical path. + private void throwOOBException() { + throw new IndexOutOfBoundsException( + String.format("size: %d, address %s, addressLimit %d", size, address, addressLimit)); + } + + // ------------------------------------------------------------------------- + // Write Methods + // ------------------------------------------------------------------------- + + /** Returns the {@code writerIndex} of this buffer. */ + public int writerIndex() { + return writerIndex; + } + + /** + * Sets the {@code writerIndex} of this buffer. + * + * @throws IndexOutOfBoundsException if the specified {@code writerIndex} is less than {@code 0} + * or greater than {@code this.size} + */ + public void writerIndex(int writerIndex) { + if (writerIndex < 0 || writerIndex > size) { + throwOOBExceptionForWriteIndex(writerIndex); + } + this.writerIndex = writerIndex; + } + + private void throwOOBExceptionForWriteIndex(int writerIndex) { + throw new IndexOutOfBoundsException( + String.format( + "writerIndex: %d (expected: 0 <= writerIndex <= size(%d))", writerIndex, size)); + } + + // CHECKSTYLE.OFF:MethodName + public void _unsafeWriterIndex(int writerIndex) { + // CHECKSTYLE.ON:MethodName + this.writerIndex = writerIndex; + } + + /** Returns heap index for writer index if buffer is a heap buffer. */ + // CHECKSTYLE.OFF:MethodName + public int _unsafeHeapWriterIndex() { + // CHECKSTYLE.ON:MethodName + return writerIndex + heapOffset; + } + + // CHECKSTYLE.OFF:MethodName + public long _unsafeWriterAddress() { + // CHECKSTYLE.ON:MethodName + checkHeapAddressAccess(); + return address + writerIndex; + } + + // CHECKSTYLE.OFF:MethodName + public void _increaseWriterIndexUnsafe(int diff) { + // CHECKSTYLE.ON:MethodName + this.writerIndex = writerIndex + diff; + } + + /** Increase writer index and grow buffer if needed. */ + public void increaseWriterIndex(int diff) { + int writerIdx = writerIndex + diff; + ensure(writerIdx); + this.writerIndex = writerIdx; + } + + public void writeBoolean(boolean value) { + if (AndroidSupport.IS_ANDROID) { + MemoryOps.writeBoolean(this, value); + } else { + final int writerIdx = writerIndex; + final int newIdx = writerIdx + 1; + ensure(newIdx); + final long pos = address + writerIdx; + storeByte(pos, (byte) (value ? 1 : 0)); + writerIndex = newIdx; + } + } + + // CHECKSTYLE.OFF:MethodName + public void _unsafeWriteByte(byte value) { + // CHECKSTYLE.ON:MethodName + if (AndroidSupport.IS_ANDROID) { + MemoryOps.unsafeWriteByte(this, value); + } else { + final int writerIdx = writerIndex; + final int newIdx = writerIdx + 1; + final long pos = address + writerIdx; + storeByte(pos, value); + writerIndex = newIdx; + } + } + + public void writeUInt8(int value) { + writeByte((byte) value); + } + + public void writeByte(byte value) { + if (AndroidSupport.IS_ANDROID) { + MemoryOps.writeByte(this, value); + } else { + final int writerIdx = writerIndex; + final int newIdx = writerIdx + 1; + ensure(newIdx); + final long pos = address + writerIdx; + storeByte(pos, value); + writerIndex = newIdx; + } + } + + public void writeByte(int value) { + writeByte((byte) value); + } + + public void writeChar(char value) { + if (AndroidSupport.IS_ANDROID) { + MemoryOps.writeChar(this, value); + } else { + final int writerIdx = writerIndex; + final int newIdx = writerIdx + 2; + ensure(newIdx); + final long pos = address + writerIdx; + if (!LITTLE_ENDIAN) { + value = Character.reverseBytes(value); + } + storeChar(pos, value); + writerIndex = newIdx; + } + } + + public void writeInt16(short value) { + if (AndroidSupport.IS_ANDROID) { + MemoryOps.writeInt16(this, value); + } else { + final int writerIdx = writerIndex; + final int newIdx = writerIdx + 2; + ensure(newIdx); + if (!LITTLE_ENDIAN) { + value = Short.reverseBytes(value); + } + storeShort(address + writerIdx, value); + writerIndex = newIdx; + } + } + + public void writeInt32(int value) { + if (AndroidSupport.IS_ANDROID) { + MemoryOps.writeInt32(this, value); + } else { + final int writerIdx = writerIndex; + final int newIdx = writerIdx + 4; + ensure(newIdx); + if (!LITTLE_ENDIAN) { + value = Integer.reverseBytes(value); + } + storeInt(address + writerIdx, value); + writerIndex = newIdx; + } + } + + public void writeInt64(long value) { + if (AndroidSupport.IS_ANDROID) { + MemoryOps.writeInt64(this, value); + } else { + final int writerIdx = writerIndex; + final int newIdx = writerIdx + 8; + ensure(newIdx); + if (!LITTLE_ENDIAN) { + value = Long.reverseBytes(value); + } + storeLong(address + writerIdx, value); + writerIndex = newIdx; + } + } + + public void writeFloat32(float value) { + if (AndroidSupport.IS_ANDROID) { + MemoryOps.writeFloat32(this, value); + } else { + final int writerIdx = writerIndex; + final int newIdx = writerIdx + 4; + ensure(newIdx); + int v = Float.floatToRawIntBits(value); + if (!LITTLE_ENDIAN) { + v = Integer.reverseBytes(v); + } + storeInt(address + writerIdx, v); + writerIndex = newIdx; + } + } + + public void writeFloat64(double value) { + if (AndroidSupport.IS_ANDROID) { + MemoryOps.writeFloat64(this, value); + } else { + final int writerIdx = writerIndex; + final int newIdx = writerIdx + 8; + ensure(newIdx); + long v = Double.doubleToRawLongBits(value); + if (!LITTLE_ENDIAN) { + v = Long.reverseBytes(v); + } + storeLong(address + writerIdx, v); + writerIndex = newIdx; + } + } + + /** + * Write int using variable length encoding. If the value is positive, use {@link #writeVarUInt32} + * to save one bit. + */ + public int writeVarInt32(int v) { + if (AndroidSupport.IS_ANDROID) { + return MemoryOps.writeVarInt32(this, v); + } + ensure(writerIndex + 8); + // Zigzag encoding: maps negative values to positive values + // This works entirely in int without conversion to long + int varintBytes = _unsafePutVarUInt32(writerIndex, (v << 1) ^ (v >> 31)); + writerIndex += varintBytes; + return varintBytes; + } + + /** + * For implementation efficiency, this method needs at most 8 bytes for writing 5 bytes using long + * to avoid using two memory operations. + */ + @CodegenInvoke + // CHECKSTYLE.OFF:MethodName + public int _unsafeWriteVarInt32(int v) { + // CHECKSTYLE.ON:MethodName + if (AndroidSupport.IS_ANDROID) { + return MemoryOps.unsafeWriteVarInt32(this, v); + } + // Zigzag encoding ensures negatives close to zero are encoded in few bytes + int varintBytes = _unsafePutVarUInt32(writerIndex, (v << 1) ^ (v >> 31)); + writerIndex += varintBytes; + return varintBytes; + } + + /** + * Writes a 1-5 byte int. + * + * @return The number of bytes written. + */ + public int writeVarUInt32(int v) { + if (AndroidSupport.IS_ANDROID) { + return MemoryOps.writeVarUInt32(this, v); + } + // ensure at least 8 bytes are writable at once, so jvm-jit + // generated code is smaller. Otherwise, the reference writer fast path + // may be `callee is too large`/`already compiled into a big method` + ensure(writerIndex + 8); + int varintBytes = _unsafePutVarUInt32(writerIndex, v); + writerIndex += varintBytes; + return varintBytes; + } + + /** + * For implementation efficiency, this method needs at most 8 bytes for writing 5 bytes using long + * to avoid using two memory operations. + */ + // CHECKSTYLE.OFF:MethodName + public int _unsafeWriteVarUInt32(int v) { + // CHECKSTYLE.ON:MethodName + if (AndroidSupport.IS_ANDROID) { + return MemoryOps.unsafeWriteVarUInt32(this, v); + } + int varintBytes = _unsafePutVarUInt32(writerIndex, v); + writerIndex += varintBytes; + return varintBytes; + } + + /** + * Fast method for write an unsigned varint which is mostly a small value in 7 bits value in [0, + * 127). When the value is equal or greater than 127, the write will be a little slower. + */ + public int writeVarUInt32Small7(int value) { + if (AndroidSupport.IS_ANDROID) { + return MemoryOps.writeVarUInt32Small7(this, value); + } + ensure(writerIndex + 8); + if (value >>> 7 == 0) { + storeByte(address + writerIndex++, (byte) value); + return 1; + } + return continueWriteVarUInt32Small7(value); + } + + // Generated serializers depend on these small-varint JVM paths staying small enough for C2 + // inlining. Android exits through MemoryOps above; keep little-endian 1/2-byte stores local and + // move only cold 3+ byte or big-endian cases into helpers. + private int continueWriteVarUInt32Small7(int value) { + int writerIdx = writerIndex; + byte[] heap = heapMemory; + if (heap != null) { + int diff = putVarUInt32Heap(heap, (int) (address + writerIdx), value); + writerIndex += diff; + return diff; + } + ByteBuffer direct = nativeOffHeapBuffer; + if (direct != null) { + int diff = putVarUInt32Direct(direct, writerIdx, value); + writerIndex += diff; + return diff; + } + int encoded = (value & 0x7F); + encoded |= (((value & 0x3f80) << 1) | 0x80); + if (!LITTLE_ENDIAN) { + int diff = putVarUInt32BigEndian(writerIdx, encoded, value); + writerIndex += diff; + return diff; + } + if (value >>> 14 == 0) { + storeInt(address + writerIdx, encoded); + writerIndex += 2; + return 2; + } + int diff = continuePutVarUInt32(writerIdx, encoded, value); + writerIndex += diff; + return diff; + } + + /** + * Writes an unsigned 32-bit varint at the given index using int operations. Caller must ensure + * there are at least 8 bytes available for writing. This method avoids int-to-long conversion + * overhead for the common cases (1-4 bytes). + * + * @param index the position to write at + * @param value the unsigned 32-bit value (high bit may be set) + * @return the number of bytes written (1-5) + */ + // CHECKSTYLE.OFF:MethodName + public int _unsafePutVarUInt32(int index, int value) { + // CHECKSTYLE.ON:MethodName + if (AndroidSupport.IS_ANDROID) { + return MemoryOps.unsafePutVarUInt32(this, index, value); + } + byte[] heap = heapMemory; + if (heap != null) { + return putVarUInt32Heap(heap, (int) (address + index), value); + } + ByteBuffer direct = nativeOffHeapBuffer; + if (direct != null) { + return putVarUInt32Direct(direct, index, value); + } + int encoded = (value & 0x7F); + if (value >>> 7 == 0) { + storeByte(address + index, (byte) value); + return 1; + } + // bit 8 `set` indicates have next data bytes. + // 0x3f80: 0b1111111 << 7 + encoded |= (((value & 0x3f80) << 1) | 0x80); + if (!LITTLE_ENDIAN) { + return putVarUInt32BigEndian(index, encoded, value); + } + if (value >>> 14 == 0) { + storeInt(address + index, encoded); + return 2; + } + return continuePutVarUInt32(index, encoded, value); + } + + private int putVarUInt32Direct(ByteBuffer direct, int index, int value) { + int pos = (int) (address + index); + int encoded = (value & 0x7F); + if (value >>> 7 == 0) { + direct.put(pos, (byte) value); + return 1; + } + encoded |= (((value & 0x3f80) << 1) | 0x80); + if (!LITTLE_ENDIAN) { + return putVarUInt32BigEndian(index, encoded, value); + } + if (value >>> 14 == 0) { + BYTE_BUFFER_INT.set(direct, pos, encoded); + return 2; + } + return continuePutVarUInt32Direct(direct, pos, encoded, value); + } + + private static int continuePutVarUInt32Direct( + ByteBuffer direct, int pos, int encoded, int value) { + encoded |= (((value & 0x1fc000) << 2) | 0x8000); + if (value >>> 21 == 0) { + BYTE_BUFFER_INT.set(direct, pos, encoded); + return 3; + } + encoded |= ((value & 0xfe00000) << 3) | 0x800000; + if (value >>> 28 == 0) { + BYTE_BUFFER_INT.set(direct, pos, encoded); + return 4; + } + long encodedLong = Integer.toUnsignedLong(encoded) | 0x80000000L; + encodedLong |= (long) (value >>> 28) << 32; + BYTE_BUFFER_LONG.set(direct, pos, encodedLong); + return 5; + } + + private int continuePutVarUInt32(int index, int encoded, int value) { + // 0x1fc000: 0b1111111 << 14 + encoded |= (((value & 0x1fc000) << 2) | 0x8000); + if (value >>> 21 == 0) { + storeInt(address + index, encoded); + return 3; + } + // 0xfe00000: 0b1111111 << 21 + encoded |= ((value & 0xfe00000) << 3) | 0x800000; + if (value >>> 28 == 0) { + storeInt(address + index, encoded); + return 4; + } + // 5-byte case: bits 28-31 go to the 5th byte + // Need long for the final write to include the 5th byte + long encodedLong = Integer.toUnsignedLong(encoded) | 0x80000000L; + encodedLong |= (long) (value >>> 28) << 32; + storeLong(address + index, encodedLong); + return 5; + } + + private int putVarUInt32BigEndian(int index, int encoded, int value) { + if (value >>> 14 == 0) { + storeInt(address + index, Integer.reverseBytes(encoded)); + return 2; + } + return continuePutVarUInt32BigEndian(index, encoded, value); + } + + private int continuePutVarUInt32BigEndian(int index, int encoded, int value) { + // 0x1fc000: 0b1111111 << 14 + encoded |= (((value & 0x1fc000) << 2) | 0x8000); + if (value >>> 21 == 0) { + storeInt(address + index, Integer.reverseBytes(encoded)); + return 3; + } + // 0xfe00000: 0b1111111 << 21 + encoded |= ((value & 0xfe00000) << 3) | 0x800000; + if (value >>> 28 == 0) { + storeInt(address + index, Integer.reverseBytes(encoded)); + return 4; + } + // 5-byte case: bits 28-31 go to the 5th byte + // Need long for the final write to include the 5th byte + long encodedLong = Integer.toUnsignedLong(encoded) | 0x80000000L; + encodedLong |= (long) (value >>> 28) << 32; + storeLong(address + index, Long.reverseBytes(encodedLong)); + return 5; + } + + /** + * Caller must ensure there must be at least 8 bytes for writing, otherwise the crash may occur. + * Don't pass int value to avoid sign extension. + */ + // CHECKSTYLE.OFF:MethodName + public int _unsafePutVarUint36Small(int index, long value) { + // CHECKSTYLE.ON:MethodName + if (AndroidSupport.IS_ANDROID) { + return MemoryOps.unsafePutVarUint36Small(this, index, value); + } + long encoded = (value & 0x7F); + if (value >>> 7 == 0) { + storeByte(address + index, (byte) value); + return 1; + } + // bit 8 `set` indicates have next data bytes. + // 0x3f80: 0b1111111 << 7 + encoded |= (((value & 0x3f80) << 1) | 0x80); + if (!LITTLE_ENDIAN) { + return putVarUint36SmallBigEndian(index, encoded, value); + } + if (value >>> 14 == 0) { + storeInt(address + index, (int) encoded); + return 2; + } + return continuePutVarUint36Small(index, encoded, value); + } + + private int continuePutVarUint36Small(int index, long encoded, long value) { + // 0x1fc000: 0b1111111 << 14 + encoded |= (((value & 0x1fc000) << 2) | 0x8000); + if (value >>> 21 == 0) { + storeInt(address + index, (int) encoded); + return 3; + } + // 0xfe00000: 0b1111111 << 21 + encoded |= ((value & 0xfe00000) << 3) | 0x800000; + if (value >>> 28 == 0) { + storeInt(address + index, (int) encoded); + return 4; + } + // 0xff0000000: 0b11111111 << 28. Note eight `1` here instead of seven. + encoded |= ((value & 0xff0000000L) << 4) | 0x80000000L; + storeLong(address + index, encoded); + return 5; + } + + private int putVarUint36SmallBigEndian(int index, long encoded, long value) { + if (value >>> 14 == 0) { + storeInt(address + index, Integer.reverseBytes((int) encoded)); + return 2; + } + return continuePutVarUint36SmallBigEndian(index, encoded, value); + } + + private int continuePutVarUint36SmallBigEndian(int index, long encoded, long value) { + // 0x1fc000: 0b1111111 << 14 + encoded |= (((value & 0x1fc000) << 2) | 0x8000); + if (value >>> 21 == 0) { + storeInt(address + index, Integer.reverseBytes((int) encoded)); + return 3; + } + // 0xfe00000: 0b1111111 << 21 + encoded |= ((value & 0xfe00000) << 3) | 0x800000; + if (value >>> 28 == 0) { + storeInt(address + index, Integer.reverseBytes((int) encoded)); + return 4; + } + // 0xff0000000: 0b11111111 << 28. Note eight `1` here instead of seven. + encoded |= ((value & 0xff0000000L) << 4) | 0x80000000L; + storeLong(address + index, Long.reverseBytes(encoded)); + return 5; + } + + /** + * Writes a 1-9 byte int, padding necessary bytes to align `writerIndex` to 4-byte. + * + * @return The number of bytes written. + */ + public int writeVarUInt32Aligned(int value) { + if (AndroidSupport.IS_ANDROID) { + return MemoryOps.writeVarUInt32Aligned(this, value); + } + // Mask first 6 bits, + // bit 7 `unset` indicates have next padding bytes, + // bit 8 `set` indicates have next data bytes. + if (value >>> 6 == 0) { + return writeVarUInt32Aligned1(value); + } + if (value >>> 12 == 0) { // 2 byte data + return writeVarUInt32Aligned2(value); + } + if (value >>> 18 == 0) { // 3 byte data + return writeVarUInt32Aligned3(value); + } + if (value >>> 24 == 0) { // 4 byte data + return writeVarUInt32Aligned4(value); + } + if (value >>> 30 == 0) { // 5 byte data + return writeVarUInt32Aligned5(value); + } + // 6 byte data + return writeVarUInt32Aligned6(value); + } + + private int writeVarUInt32Aligned1(int value) { + final int writerIdx = writerIndex; + int numPaddingBytes = 4 - writerIdx % 4; + ensure(writerIdx + 5); // 1 byte + 4 bytes(zero out), padding range in (zero out) + int first = (value & 0x3F); + final long pos = address + writerIdx; + if (numPaddingBytes == 1) { + // bit 7 `set` indicates not have padding bytes. + // bit 8 `set` indicates have next data bytes. + storeByte(pos, (byte) (first | 0x40)); + writerIndex = (writerIdx + 1); + return 1; + } else { + storeByte(pos, (byte) first); + // zero out 4 bytes, so that `bit 7` value can be trusted. + storeInt(pos + 1, 0); + storeByte(pos + numPaddingBytes - 1, (byte) (0x40)); + writerIndex = writerIdx + numPaddingBytes; + return numPaddingBytes; + } + } + + private int writeVarUInt32Aligned2(int value) { + final int writerIdx = writerIndex; + int numPaddingBytes = 4 - writerIdx % 4; + ensure(writerIdx + 6); // 2 byte + 4 bytes(zero out), padding range in (zero out) + int first = (value & 0x3F); + final long pos = address + writerIdx; + storeByte(pos, (byte) (first | 0x80)); + if (numPaddingBytes == 2) { + // bit 7 `set` indicates not have padding bytes. + // bit 8 `set` indicates have next data bytes. + storeByte(pos + 1, (byte) ((value >>> 6) | 0x40)); + writerIndex = writerIdx + 2; + return 2; + } else { + storeByte(pos + 1, (byte) (value >>> 6)); + // zero out 4 bytes, so that `bit 7` value can be trusted. + storeInt(pos + 2, 0); + if (numPaddingBytes > 2) { + storeByte(pos + numPaddingBytes - 1, (byte) (0x40)); + writerIndex = writerIdx + numPaddingBytes; + return numPaddingBytes; + } else { + storeByte(pos + 4, (byte) (0x40)); + writerIndex = writerIdx + numPaddingBytes + 4; + return numPaddingBytes + 4; + } + } + } + + private int writeVarUInt32Aligned3(int value) { + final int writerIdx = writerIndex; + int numPaddingBytes = 4 - writerIdx % 4; + ensure(writerIdx + 7); // 3 byte + 4 bytes(zero out), padding range in (zero out) + int first = (value & 0x3F); + final long pos = address + writerIdx; + storeByte(pos, (byte) (first | 0x80)); + storeByte(pos + 1, (byte) ((value >>> 6) | 0x80)); + if (numPaddingBytes == 3) { + // bit 7 `set` indicates not have padding bytes. + // bit 8 `set` indicates have next data bytes. + storeByte(pos + 2, (byte) ((value >>> 12) | 0x40)); + writerIndex = writerIdx + 3; + return 3; + } else { + storeByte(pos + 2, (byte) (value >>> 12)); + // zero out 4 bytes, so that `bit 7` value can be trusted. + storeInt(pos + 3, 0); + if (numPaddingBytes == 4) { + storeByte(pos + numPaddingBytes - 1, (byte) (0x40)); + writerIndex = writerIdx + numPaddingBytes; + return numPaddingBytes; + } else { + storeByte(pos + numPaddingBytes + 3, (byte) (0x40)); + writerIndex = writerIdx + numPaddingBytes + 4; + return numPaddingBytes + 4; + } + } + } + + private int writeVarUInt32Aligned4(int value) { + final int writerIdx = writerIndex; + int numPaddingBytes = 4 - writerIdx % 4; + ensure(writerIdx + 8); // 4 byte + 4 bytes(zero out), padding range in (zero out) + int first = (value & 0x3F); + final long pos = address + writerIdx; + storeByte(pos, (byte) (first | 0x80)); + storeByte(pos + 1, (byte) (value >>> 6 | 0x80)); + storeByte(pos + 2, (byte) (value >>> 12 | 0x80)); + if (numPaddingBytes == 4) { + // bit 7 `set` indicates not have padding bytes. + // bit 8 `set` indicates have next data bytes. + storeByte(pos + 3, (byte) ((value >>> 18) | 0x40)); + writerIndex = writerIdx + 4; + return 4; + } else { + storeByte(pos + 3, (byte) (value >>> 18)); + // zero out 4 bytes, so that `bit 7` value can be trusted. + storeInt(pos + 4, 0); + storeByte(pos + numPaddingBytes + 3, (byte) (0x40)); + writerIndex = writerIdx + numPaddingBytes + 4; + return numPaddingBytes + 4; + } + } + + private int writeVarUInt32Aligned5(int value) { + final int writerIdx = writerIndex; + int numPaddingBytes = 4 - writerIdx % 4; + ensure(writerIdx + 9); // 5 byte + 4 bytes(zero out), padding range in (zero out) + int first = (value & 0x3F); + final long pos = address + writerIdx; + storeByte(pos, (byte) (first | 0x80)); + storeByte(pos + 1, (byte) (value >>> 6 | 0x80)); + storeByte(pos + 2, (byte) (value >>> 12 | 0x80)); + storeByte(pos + 3, (byte) (value >>> 18 | 0x80)); + if (numPaddingBytes == 1) { + // bit 7 `set` indicates not have padding bytes. + // bit 8 `set` indicates have next data bytes. + storeByte(pos + 4, (byte) ((value >>> 24) | 0x40)); + writerIndex = writerIdx + 5; + return 5; + } else { + storeByte(pos + 4, (byte) (value >>> 24)); + // zero out 4 bytes, so that `bit 7` value can be trusted. + storeInt(pos + 5, 0); + storeByte(pos + numPaddingBytes + 3, (byte) (0x40)); + writerIndex = writerIdx + numPaddingBytes + 4; + return numPaddingBytes + 4; + } + } + + private int writeVarUInt32Aligned6(int value) { + final int writerIdx = writerIndex; + int numPaddingBytes = 4 - writerIdx % 4; + ensure(writerIdx + 10); // 6 byte + 4 bytes(zero out), padding range in (zero out) + int first = (value & 0x3F); + final long pos = address + writerIdx; + storeByte(pos, (byte) (first | 0x80)); + storeByte(pos + 1, (byte) (value >>> 6 | 0x80)); + storeByte(pos + 2, (byte) (value >>> 12 | 0x80)); + storeByte(pos + 3, (byte) (value >>> 18 | 0x80)); + storeByte(pos + 4, (byte) (value >>> 24 | 0x80)); + if (numPaddingBytes == 2) { + // bit 7 `set` indicates not have padding bytes. + // bit 8 `set` indicates have next data bytes. + storeByte(pos + 5, (byte) ((value >>> 30) | 0x40)); + writerIndex = writerIdx + 6; + return 6; + } else { + storeByte(pos + 5, (byte) (value >>> 30)); + // zero out 4 bytes, so that `bit 7` value can be trusted. + storeInt(pos + 6, 0); + if (numPaddingBytes == 1) { + storeByte(pos + 8, (byte) (0x40)); + writerIndex = writerIdx + 9; + return 9; + } else { + storeByte(pos + numPaddingBytes + 3, (byte) (0x40)); + writerIndex = writerIdx + numPaddingBytes + 4; + return numPaddingBytes + 4; + } + } + } + + /** + * Write long using variable length encoding. If the value is positive, use {@link + * #writeVarUInt64} to save one bit. + */ + public int writeVarInt64(long value) { + if (AndroidSupport.IS_ANDROID) { + return MemoryOps.writeVarInt64(this, value); + } + ensure(writerIndex + 9); + return _unsafeWriteVarUInt64((value << 1) ^ (value >> 63)); + } + + @CodegenInvoke + // CHECKSTYLE.OFF:MethodName + public int _unsafeWriteVarInt64(long value) { + // CHECKSTYLE.ON:MethodName + if (AndroidSupport.IS_ANDROID) { + return MemoryOps.unsafeWriteVarInt64(this, value); + } + return _unsafeWriteVarUInt64((value << 1) ^ (value >> 63)); + } + + public int writeVarUInt64(long value) { + if (AndroidSupport.IS_ANDROID) { + return MemoryOps.writeVarUInt64(this, value); + } + // Var long encoding algorithm is based kryo UnsafeMemoryOutput.writeVarInt64. + // var long are written using little endian byte order. + ensure(writerIndex + 9); + return _unsafeWriteVarUInt64(value); + } + + // CHECKSTYLE.OFF:MethodName + @CodegenInvoke + public int _unsafeWriteVarUInt64(long value) { + // CHECKSTYLE.ON:MethodName + if (AndroidSupport.IS_ANDROID) { + return MemoryOps.unsafeWriteVarUInt64(this, value); + } + final int writerIndex = this.writerIndex; + int varInt; + varInt = (int) (value & 0x7F); + if (value >>> 7 == 0) { + storeByte(address + writerIndex, (byte) varInt); + this.writerIndex = writerIndex + 1; + return 1; + } + varInt |= (int) (((value & 0x3f80) << 1) | 0x80); + if (value >>> 14 == 0) { + _unsafePutInt32(writerIndex, varInt); + this.writerIndex = writerIndex + 2; + return 2; + } + varInt |= (int) (((value & 0x1fc000) << 2) | 0x8000); + if (value >>> 21 == 0) { + _unsafePutInt32(writerIndex, varInt); + this.writerIndex = writerIndex + 3; + return 3; + } + varInt |= (int) (((value & 0xfe00000) << 3) | 0x800000); + if (value >>> 28 == 0) { + _unsafePutInt32(writerIndex, varInt); + this.writerIndex = writerIndex + 4; + return 4; + } + long varLong = (varInt & 0xFFFFFFFFL); + varLong |= ((value & 0x7f0000000L) << 4) | 0x80000000L; + if (value >>> 35 == 0) { + _unsafePutInt64(writerIndex, varLong); + this.writerIndex = writerIndex + 5; + return 5; + } + varLong |= ((value & 0x3f800000000L) << 5) | 0x8000000000L; + if (value >>> 42 == 0) { + _unsafePutInt64(writerIndex, varLong); + this.writerIndex = writerIndex + 6; + return 6; + } + varLong |= ((value & 0x1fc0000000000L) << 6) | 0x800000000000L; + if (value >>> 49 == 0) { + _unsafePutInt64(writerIndex, varLong); + this.writerIndex = writerIndex + 7; + return 7; + } + varLong |= ((value & 0xfe000000000000L) << 7) | 0x80000000000000L; + value >>>= 56; + if (value == 0) { + _unsafePutInt64(writerIndex, varLong); + this.writerIndex = writerIndex + 8; + return 8; + } + _unsafePutInt64(writerIndex, varLong | 0x8000000000000000L); + storeByte(address + writerIndex + 8, (byte) (value & 0xFF)); + this.writerIndex = writerIndex + 9; + return 9; + } + + /** + * Write signed long using fory Tagged(Small long as int) encoding. If long is in [0xc0000000, + * 0x3fffffff], encode as 4 bytes int: {@code | little-endian: ((int) value) << 1 |}; Otherwise + * write as 9 bytes: {@code | 0b1 | little-endian 8bytes long |}. + */ + public int writeTaggedInt64(long value) { + ensure(writerIndex + 9); + return _unsafeWriteTaggedInt64(value); + } + + /** + * Write unsigned long using fory Tagged(Small long as int) encoding. If long is in [0, + * 0x7fffffff], encode as 4 bytes int: {@code | little-endian: ((int) value) << 1 |}; Otherwise + * write as 9 bytes: {@code | 0b1 | little-endian 8bytes long |}. + */ + public int writeTaggedUInt64(long value) { + ensure(writerIndex + 9); + return _unsafeWriteTaggedUInt64(value); + } + + /** Write unsigned long using fory Tagged(Small Long as Int) encoding. */ + // CHECKSTYLE.OFF:MethodName + public int _unsafeWriteTaggedUInt64(long value) { + // CHECKSTYLE.ON:MethodName + if (AndroidSupport.IS_ANDROID) { + return MemoryOps.unsafeWriteTaggedUInt64(this, value); + } + final int writerIndex = this.writerIndex; + final long pos = address + writerIndex; + final byte[] heapMemory = this.heapMemory; + if (value >= 0 && value <= Integer.MAX_VALUE) { + int v = ((int) value) << 1; // bit 0 unset, means int. + if (!LITTLE_ENDIAN) { + v = Integer.reverseBytes(v); + } + storeInt(pos, v); + this.writerIndex = writerIndex + 4; + return 4; + } else { + storeByte(pos, BIG_LONG_FLAG); + if (!LITTLE_ENDIAN) { + value = Long.reverseBytes(value); + } + storeLong(pos + 1, value); + this.writerIndex = writerIndex + 9; + return 9; + } + } + + private static final long HALF_MAX_INT_VALUE = Integer.MAX_VALUE / 2; + private static final long HALF_MIN_INT_VALUE = Integer.MIN_VALUE / 2; + private static final byte BIG_LONG_FLAG = 0b1; // bit 0 set, means big long. + + /** Write long using fory Tagged(Small Long as Int) encoding. */ + // CHECKSTYLE.OFF:MethodName + public int _unsafeWriteTaggedInt64(long value) { + // CHECKSTYLE.ON:MethodName + if (AndroidSupport.IS_ANDROID) { + return MemoryOps.unsafeWriteTaggedInt64(this, value); + } + final int writerIndex = this.writerIndex; + final long pos = address + writerIndex; + final byte[] heapMemory = this.heapMemory; + if (value >= HALF_MIN_INT_VALUE && value <= HALF_MAX_INT_VALUE) { + // write: + // 00xxx -> 0xxx + // 11xxx -> 1xxx + // read: + // 0xxx -> 00xxx + // 1xxx -> 11xxx + int v = ((int) value) << 1; // bit 0 unset, means int. + if (!LITTLE_ENDIAN) { + v = Integer.reverseBytes(v); + } + storeInt(pos, v); + this.writerIndex = writerIndex + 4; + return 4; + } else { + storeByte(pos, BIG_LONG_FLAG); + if (!LITTLE_ENDIAN) { + value = Long.reverseBytes(value); + } + storeLong(pos + 1, value); + this.writerIndex = writerIndex + 9; + return 9; + } + } + + public void writeBytes(byte[] bytes) { + writeBytes(bytes, 0, bytes.length); + } + + public void writeBytes(byte[] bytes, int offset, int length) { + final int writerIdx = writerIndex; + final int newIdx = writerIdx + length; + ensure(newIdx); + put(writerIdx, bytes, offset, length); + writerIndex = newIdx; + } + + public void write(ByteBuffer source) { + write(source, source.remaining()); + } + + public void write(ByteBuffer source, int numBytes) { + final int writerIdx = writerIndex; + final int newIdx = writerIdx + numBytes; + ensure(newIdx); + put(writerIdx, source, numBytes); + writerIndex = newIdx; + } + + public void writeBytesWithSize(byte[] values) { + if (AndroidSupport.IS_ANDROID) { + MemoryOps.writeBytesWithSize(this, values); + } else { + writeVarUInt32Small7(values.length); + writeBytes(values, 0, values.length); + } + } + + public void writeBooleansWithSize(boolean[] values) { + if (AndroidSupport.IS_ANDROID) { + MemoryOps.writeBooleansWithSize(this, values); + } else { + writeVarUInt32Small7(values.length); + writeBooleans(values, 0, values.length); + } + } + + public void writeBooleans(boolean[] values) { + writeBooleans(values, 0, values.length); + } + + public void writeBooleans(boolean[] values, int offset, int numElements) { + if (AndroidSupport.IS_ANDROID) { + MemoryOps.writeBooleans(this, values, offset, numElements); + } else { + final int writerIdx = writerIndex; + final int newIdx = writerIdx + numElements; + ensure(newIdx); + memoryAccess.copyMemory( + values, + UnsafeOps.BOOLEAN_ARRAY_OFFSET + offset, + heapMemory, + address + writerIdx, + numElements); + writerIndex = newIdx; + } + } + + public void writeCharsWithSize(char[] values) { + if (AndroidSupport.IS_ANDROID) { + MemoryOps.writeCharsWithSize(this, values); + } else { + int numBytes = Math.multiplyExact(values.length, 2); + writeVarUInt32Small7(numBytes); + writeChars(values, 0, values.length); + } + } + + public void writeChars(char[] values) { + writeChars(values, 0, values.length); + } + + public void writeChars(char[] values, int offset, int numElements) { + if (AndroidSupport.IS_ANDROID) { + MemoryOps.writeChars(this, values, offset, numElements); + } else { + int numBytes = Math.multiplyExact(numElements, 2); + final int writerIdx = writerIndex; + final int newIdx = writerIdx + numBytes; + ensure(newIdx); + memoryAccess.copyMemory( + values, + UnsafeOps.CHAR_ARRAY_OFFSET + ((long) offset << 1), + heapMemory, + address + writerIdx, + numBytes); + writerIndex = newIdx; + } + } + + public void writeShortsWithSize(short[] values) { + if (AndroidSupport.IS_ANDROID) { + MemoryOps.writeShortsWithSize(this, values); + } else { + int numBytes = Math.multiplyExact(values.length, 2); + writeVarUInt32Small7(numBytes); + writeShorts(values, 0, values.length); + } + } + + public void writeShorts(short[] values) { + writeShorts(values, 0, values.length); + } + + public void writeShorts(short[] values, int offset, int numElements) { + if (AndroidSupport.IS_ANDROID) { + MemoryOps.writeShorts(this, values, offset, numElements); + } else { + int numBytes = Math.multiplyExact(numElements, 2); + final int writerIdx = writerIndex; + final int newIdx = writerIdx + numBytes; + ensure(newIdx); + memoryAccess.copyMemory( + values, + UnsafeOps.SHORT_ARRAY_OFFSET + ((long) offset << 1), + heapMemory, + address + writerIdx, + numBytes); + writerIndex = newIdx; + } + } + + public void writeIntsWithSize(int[] values) { + if (AndroidSupport.IS_ANDROID) { + MemoryOps.writeIntsWithSize(this, values); + } else { + int numBytes = Math.multiplyExact(values.length, 4); + writeVarUInt32Small7(numBytes); + writeInts(values, 0, values.length); + } + } + + public void writeInts(int[] values) { + writeInts(values, 0, values.length); + } + + public void writeInts(int[] values, int offset, int numElements) { + if (AndroidSupport.IS_ANDROID) { + MemoryOps.writeInts(this, values, offset, numElements); + } else { + int numBytes = Math.multiplyExact(numElements, 4); + final int writerIdx = writerIndex; + final int newIdx = writerIdx + numBytes; + ensure(newIdx); + memoryAccess.copyMemory( + values, + UnsafeOps.INT_ARRAY_OFFSET + ((long) offset << 2), + heapMemory, + address + writerIdx, + numBytes); + writerIndex = newIdx; + } + } + + public void writeLongsWithSize(long[] values) { + if (AndroidSupport.IS_ANDROID) { + MemoryOps.writeLongsWithSize(this, values); + } else { + int numBytes = Math.multiplyExact(values.length, 8); + writeVarUInt32Small7(numBytes); + writeLongs(values, 0, values.length); + } + } + + public void writeLongs(long[] values) { + writeLongs(values, 0, values.length); + } + + public void writeLongs(long[] values, int offset, int numElements) { + if (AndroidSupport.IS_ANDROID) { + MemoryOps.writeLongs(this, values, offset, numElements); + } else { + int numBytes = Math.multiplyExact(numElements, 8); + final int writerIdx = writerIndex; + final int newIdx = writerIdx + numBytes; + ensure(newIdx); + memoryAccess.copyMemory( + values, + UnsafeOps.LONG_ARRAY_OFFSET + ((long) offset << 3), + heapMemory, + address + writerIdx, + numBytes); + writerIndex = newIdx; + } + } + + public void writeFloatsWithSize(float[] values) { + if (AndroidSupport.IS_ANDROID) { + MemoryOps.writeFloatsWithSize(this, values); + } else { + int numBytes = Math.multiplyExact(values.length, 4); + writeVarUInt32Small7(numBytes); + writeFloats(values, 0, values.length); + } + } + + public void writeFloats(float[] values) { + writeFloats(values, 0, values.length); + } + + public void writeFloats(float[] values, int offset, int numElements) { + if (AndroidSupport.IS_ANDROID) { + MemoryOps.writeFloats(this, values, offset, numElements); + } else { + int numBytes = Math.multiplyExact(numElements, 4); + final int writerIdx = writerIndex; + final int newIdx = writerIdx + numBytes; + ensure(newIdx); + memoryAccess.copyMemory( + values, + UnsafeOps.FLOAT_ARRAY_OFFSET + ((long) offset << 2), + heapMemory, + address + writerIdx, + numBytes); + writerIndex = newIdx; + } + } + + public void writeDoublesWithSize(double[] values) { + if (AndroidSupport.IS_ANDROID) { + MemoryOps.writeDoublesWithSize(this, values); + } else { + int numBytes = Math.multiplyExact(values.length, 8); + writeVarUInt32Small7(numBytes); + writeDoubles(values, 0, values.length); + } + } + + public void writeDoubles(double[] values) { + writeDoubles(values, 0, values.length); + } + + public void writeDoubles(double[] values, int offset, int numElements) { + if (AndroidSupport.IS_ANDROID) { + MemoryOps.writeDoubles(this, values, offset, numElements); + } else { + int numBytes = Math.multiplyExact(numElements, 8); + final int writerIdx = writerIndex; + final int newIdx = writerIdx + numBytes; + ensure(newIdx); + memoryAccess.copyMemory( + values, + UnsafeOps.DOUBLE_ARRAY_OFFSET + ((long) offset << 3), + heapMemory, + address + writerIdx, + numBytes); + writerIndex = newIdx; + } + } + + /** For off-heap buffer, this will make a heap buffer internally. */ + public void grow(int neededSize) { + int length = writerIndex + neededSize; + if (length > size) { + globalAllocator.grow(this, length); + } + } + + /** For off-heap buffer, this will make a heap buffer internally. */ + public void ensure(int length) { + if (length > size) { + globalAllocator.grow(this, length); + } + } + + // ------------------------------------------------------------------------- + // Read Methods + // ------------------------------------------------------------------------- + + // Check should be done outside to avoid this method got into the critical path. + private void throwIndexOOBExceptionForRead() { + throw new IndexOutOfBoundsException( + String.format( + "readerIndex: %d (expected: 0 <= readerIndex <= size(%d))", readerIndex, size)); + } + + // Check should be done outside to avoid this method got into the critical path. + private void throwIndexOOBExceptionForRead(int length) { + throw new IndexOutOfBoundsException( + String.format( + "readerIndex: %d (expected: 0 <= readerIndex <= size(%d)), length %d", + readerIndex, size, length)); + } + + /** Returns the {@code readerIndex} of this buffer. */ + public int readerIndex() { + return readerIndex; + } + + /** + * Sets the {@code readerIndex} of this buffer. + * + * @throws IndexOutOfBoundsException if the specified {@code readerIndex} is less than {@code 0} + * or greater than {@code this.size} + */ + public void readerIndex(int readerIndex) { + if (readerIndex < 0) { + throwIndexOOBExceptionForRead(); + } else if (readerIndex > size) { + // in this case, diff must be greater than 0. + streamReader.fillBuffer(readerIndex - size); + } + this.readerIndex = readerIndex; + } + + /** Returns array index for reader index if buffer is a heap buffer. */ + // CHECKSTYLE.OFF:MethodName + public int _unsafeHeapReaderIndex() { + // CHECKSTYLE.ON:MethodName + return readerIndex + heapOffset; + } + + // CHECKSTYLE.OFF:MethodName + public void _increaseReaderIndexUnsafe(int diff) { + // CHECKSTYLE.ON:MethodName + readerIndex += diff; + } + + public void increaseReaderIndex(int diff) { + int readerIdx = readerIndex; + readerIndex = readerIdx += diff; + if (readerIdx < 0) { + throwIndexOOBExceptionForRead(); + } else if (readerIdx > size) { + // in this case, diff must be greater than 0. + streamReader.fillBuffer(readerIdx - size); + } + } + + public long getUnsafeReaderAddress() { + checkHeapAddressAccess(); + return address + readerIndex; + } + + private void checkHeapAddressAccess() { + if (heapMemory == null) { + throw new UnsupportedOperationException( + "JDK25 MemoryBuffer does not expose raw native addresses for direct buffers."); + } + } + + public int remaining() { + return size - readerIndex; + } + + public boolean readBoolean() { + if (AndroidSupport.IS_ANDROID) { + return MemoryOps.readBoolean(this); + } + int readerIdx = readerIndex; + // use subtract to avoid overflow + if (readerIdx > size - 1) { + streamReader.fillBuffer(1); + } + readerIndex = readerIdx + 1; + return loadByte(address + readerIdx) != 0; + } + + public int readUInt8() { + if (AndroidSupport.IS_ANDROID) { + return MemoryOps.readUInt8(this); + } + int readerIdx = readerIndex; + if (readerIdx > size - 1) { + streamReader.fillBuffer(1); + } + readerIndex = readerIdx + 1; + int v = loadByte(address + readerIdx); + v &= 0b11111111; + return v; + } + + public int readUnsignedByte() { + return readUInt8(); + } + + public byte readByte() { + if (AndroidSupport.IS_ANDROID) { + return MemoryOps.readByte(this); + } + int readerIdx = readerIndex; + if (readerIdx > size - 1) { + streamReader.fillBuffer(1); + } + readerIndex = readerIdx + 1; + return loadByte(address + readerIdx); + } + + public char readChar() { + if (AndroidSupport.IS_ANDROID) { + return MemoryOps.readChar(this); + } + int readerIdx = readerIndex; + // use subtract to avoid overflow + int remaining = size - readerIdx; + if (remaining < 2) { + streamReader.fillBuffer(2 - remaining); + } + readerIndex = readerIdx + 2; + char c = loadChar(address + readerIdx); + return LITTLE_ENDIAN ? c : Character.reverseBytes(c); + } + + public short readInt16() { + if (AndroidSupport.IS_ANDROID) { + return MemoryOps.readInt16(this); + } + int readerIdx = readerIndex; + // use subtract to avoid overflow + int remaining = size - readerIdx; + if (remaining < 2) { + streamReader.fillBuffer(2 - remaining); + } + readerIndex = readerIdx + 2; + short v = loadShort(address + readerIdx); + return LITTLE_ENDIAN ? v : Short.reverseBytes(v); + } + + // Reduce method body for better inline in the caller. + @CodegenInvoke + // CHECKSTYLE.OFF:MethodName + public short _readInt16OnLE() { + // CHECKSTYLE.ON:MethodName + if (AndroidSupport.IS_ANDROID) { + return MemoryOps.readInt16(this); + } + int readerIdx = readerIndex; + // use subtract to avoid overflow + int remaining = size - readerIdx; + if (remaining < 2) { + streamReader.fillBuffer(2 - remaining); + } + readerIndex = readerIdx + 2; + return loadShort(address + readerIdx); + } + + // Reduce method body for better inline in the caller. + @CodegenInvoke + // CHECKSTYLE.OFF:MethodName + public short _readInt16OnBE() { + // CHECKSTYLE.ON:MethodName + if (AndroidSupport.IS_ANDROID) { + return MemoryOps.readInt16(this); + } + int readerIdx = readerIndex; + // use subtract to avoid overflow + int remaining = size - readerIdx; + if (remaining < 2) { + streamReader.fillBuffer(2 - remaining); + } + readerIndex = readerIdx + 2; + return Short.reverseBytes(loadShort(address + readerIdx)); + } + + public int readInt32() { + if (AndroidSupport.IS_ANDROID) { + return MemoryOps.readInt32(this); + } + int readerIdx = readerIndex; + // use subtract to avoid overflow + int remaining = size - readerIdx; + if (remaining < 4) { + streamReader.fillBuffer(4 - remaining); + } + readerIndex = readerIdx + 4; + int v = loadInt(address + readerIdx); + return LITTLE_ENDIAN ? v : Integer.reverseBytes(v); + } + + // Reduce method body for better inline in the caller. + @CodegenInvoke + // CHECKSTYLE.OFF:MethodName + public int _readInt32OnLE() { + // CHECKSTYLE.ON:MethodName + if (AndroidSupport.IS_ANDROID) { + return MemoryOps.readInt32(this); + } + int readerIdx = readerIndex; + // use subtract to avoid overflow + int remaining = size - readerIdx; + if (remaining < 4) { + streamReader.fillBuffer(4 - remaining); + } + readerIndex = readerIdx + 4; + return loadInt(address + readerIdx); + } + + // Reduce method body for better inline in the caller. + @CodegenInvoke + // CHECKSTYLE.OFF:MethodName + public int _readInt32OnBE() { + // CHECKSTYLE.ON:MethodName + if (AndroidSupport.IS_ANDROID) { + return MemoryOps.readInt32(this); + } + int readerIdx = readerIndex; + // use subtract to avoid overflow + int remaining = size - readerIdx; + if (remaining < 4) { + streamReader.fillBuffer(4 - remaining); + } + readerIndex = readerIdx + 4; + return Integer.reverseBytes(loadInt(address + readerIdx)); + } + + public long readInt64() { + if (AndroidSupport.IS_ANDROID) { + return MemoryOps.readInt64(this); + } + int readerIdx = readerIndex; + // use subtract to avoid overflow + int remaining = size - readerIdx; + if (remaining < 8) { + streamReader.fillBuffer(8 - remaining); + } + readerIndex = readerIdx + 8; + long v = loadLong(address + readerIdx); + return LITTLE_ENDIAN ? v : Long.reverseBytes(v); + } + + // Reduce method body for better inline in the caller. + @CodegenInvoke + // CHECKSTYLE.OFF:MethodName + public long _readInt64OnLE() { + // CHECKSTYLE.ON:MethodName + if (AndroidSupport.IS_ANDROID) { + return MemoryOps.readInt64(this); + } + int readerIdx = readerIndex; + // use subtract to avoid overflow + int remaining = size - readerIdx; + if (remaining < 8) { + streamReader.fillBuffer(8 - remaining); + } + readerIndex = readerIdx + 8; + return loadLong(address + readerIdx); + } + + // Reduce method body for better inline in the caller. + @CodegenInvoke + // CHECKSTYLE.OFF:MethodName + public long _readInt64OnBE() { + // CHECKSTYLE.ON:MethodName + if (AndroidSupport.IS_ANDROID) { + return MemoryOps.readInt64(this); + } + int readerIdx = readerIndex; + // use subtract to avoid overflow + int remaining = size - readerIdx; + if (remaining < 8) { + streamReader.fillBuffer(8 - remaining); + } + readerIndex = readerIdx + 8; + return Long.reverseBytes(loadLong(address + readerIdx)); + } + + /** Read signed fory Tagged(Small Long as Int) encoded long. */ + public long readTaggedInt64() { + if (LITTLE_ENDIAN) { + return _readTaggedInt64OnLE(); + } else { + return _readTaggedInt64OnBE(); + } + } + + /** Read unsigned fory Tagged(Small Long as Int) encoded long. */ + public long readTaggedUInt64() { + if (LITTLE_ENDIAN) { + return _readTaggedUInt64OnLE(); + } else { + return _readTaggedUInt64OnBE(); + } + } + + @CodegenInvoke + // CHECKSTYLE.OFF:MethodName + public long _readTaggedUInt64OnLE() { + // CHECKSTYLE.ON:MethodName + if (AndroidSupport.IS_ANDROID) { + return MemoryOps.readTaggedUInt64(this); + } + final int readIdx = readerIndex; + int diff = size - readIdx; + if (diff < 4) { + streamReader.fillBuffer(4 - diff); + } + int i = loadInt(address + readIdx); + if ((i & 0b1) != 0b1) { + readerIndex = readIdx + 4; + return i >>> 1; // unsigned right shift + } + diff = size - readIdx; + if (diff < 9) { + streamReader.fillBuffer(9 - diff); + } + readerIndex = readIdx + 9; + return loadLong(address + readIdx + 1); + } + + @CodegenInvoke + // CHECKSTYLE.OFF:MethodName + public long _readTaggedUInt64OnBE() { + // CHECKSTYLE.ON:MethodName + if (AndroidSupport.IS_ANDROID) { + return MemoryOps.readTaggedUInt64(this); + } + final int readIdx = readerIndex; + int diff = size - readIdx; + if (diff < 4) { + streamReader.fillBuffer(4 - diff); + } + int i = Integer.reverseBytes(loadInt(address + readIdx)); + if ((i & 0b1) != 0b1) { + readerIndex = readIdx + 4; + return i >>> 1; // unsigned right shift + } + diff = size - readIdx; + if (diff < 9) { + streamReader.fillBuffer(9 - diff); + } + readerIndex = readIdx + 9; + return Long.reverseBytes(loadLong(address + readIdx + 1)); + } + + @CodegenInvoke + // CHECKSTYLE.OFF:MethodName + public long _readTaggedInt64OnLE() { + // CHECKSTYLE.ON:MethodName + if (AndroidSupport.IS_ANDROID) { + return MemoryOps.readTaggedInt64(this); + } + // Duplicate and manual inline for performance. + // noinspection Duplicates + final int readIdx = readerIndex; + int diff = size - readIdx; + if (diff < 4) { + streamReader.fillBuffer(4 - diff); + } + int i = loadInt(address + readIdx); + if ((i & 0b1) != 0b1) { + readerIndex = readIdx + 4; + return i >> 1; + } + diff = size - readIdx; + if (diff < 9) { + streamReader.fillBuffer(9 - diff); + } + readerIndex = readIdx + 9; + return loadLong(address + readIdx + 1); + } + + @CodegenInvoke + // CHECKSTYLE.OFF:MethodName + public long _readTaggedInt64OnBE() { + // CHECKSTYLE.ON:MethodName + if (AndroidSupport.IS_ANDROID) { + return MemoryOps.readTaggedInt64(this); + } + // noinspection Duplicates + final int readIdx = readerIndex; + int diff = size - readIdx; + if (diff < 4) { + streamReader.fillBuffer(4 - diff); + } + int i = Integer.reverseBytes(loadInt(address + readIdx)); + if ((i & 0b1) != 0b1) { + readerIndex = readIdx + 4; + return i >> 1; + } + diff = size - readIdx; + if (diff < 9) { + streamReader.fillBuffer(9 - diff); + } + readerIndex = readIdx + 9; + return Long.reverseBytes(loadLong(address + readIdx + 1)); + } + + public float readFloat32() { + if (AndroidSupport.IS_ANDROID) { + return MemoryOps.readFloat32(this); + } + // noinspection Duplicates + int readerIdx = readerIndex; + // use subtract to avoid overflow + int remaining = size - readerIdx; + if (remaining < 4) { + streamReader.fillBuffer(4 - remaining); + } + readerIndex = readerIdx + 4; + int v = loadInt(address + readerIdx); + if (!LITTLE_ENDIAN) { + v = Integer.reverseBytes(v); + } + return Float.intBitsToFloat(v); + } + + // Reduce method body for better inline in the caller. + @CodegenInvoke + // CHECKSTYLE.OFF:MethodName + public float _readFloat32OnLE() { + // CHECKSTYLE.ON:MethodName + if (AndroidSupport.IS_ANDROID) { + return MemoryOps.readFloat32(this); + } + int readerIdx = readerIndex; + // use subtract to avoid overflow + int remaining = size - readerIdx; + if (remaining < 4) { + streamReader.fillBuffer(4 - remaining); + } + readerIndex = readerIdx + 4; + return Float.intBitsToFloat(loadInt(address + readerIdx)); + } + + // Reduce method body for better inline in the caller. + @CodegenInvoke + // CHECKSTYLE.OFF:MethodName + public float _readFloat32OnBE() { + // CHECKSTYLE.ON:MethodName + if (AndroidSupport.IS_ANDROID) { + return MemoryOps.readFloat32(this); + } + int readerIdx = readerIndex; + // use subtract to avoid overflow + int remaining = size - readerIdx; + if (remaining < 4) { + streamReader.fillBuffer(4 - remaining); + } + readerIndex = readerIdx + 4; + return Float.intBitsToFloat( + Integer.reverseBytes(loadInt(address + readerIdx))); + } + + public double readFloat64() { + if (AndroidSupport.IS_ANDROID) { + return MemoryOps.readFloat64(this); + } + // noinspection Duplicates + int readerIdx = readerIndex; + // use subtract to avoid overflow + int remaining = size - readerIdx; + if (remaining < 8) { + streamReader.fillBuffer(8 - remaining); + } + readerIndex = readerIdx + 8; + long v = loadLong(address + readerIdx); + if (!LITTLE_ENDIAN) { + v = Long.reverseBytes(v); + } + return Double.longBitsToDouble(v); + } + + // Reduce method body for better inline in the caller. + @CodegenInvoke + // CHECKSTYLE.OFF:MethodName + public double _readFloat64OnLE() { + // CHECKSTYLE.ON:MethodName + if (AndroidSupport.IS_ANDROID) { + return MemoryOps.readFloat64(this); + } + int readerIdx = readerIndex; + // use subtract to avoid overflow + int remaining = size - readerIdx; + if (remaining < 8) { + streamReader.fillBuffer(8 - remaining); + } + readerIndex = readerIdx + 8; + return Double.longBitsToDouble(loadLong(address + readerIdx)); + } + + // Reduce method body for better inline in the caller. + @CodegenInvoke + // CHECKSTYLE.OFF:MethodName + public double _readFloat64OnBE() { + // CHECKSTYLE.ON:MethodName + if (AndroidSupport.IS_ANDROID) { + return MemoryOps.readFloat64(this); + } + int readerIdx = readerIndex; + // use subtract to avoid overflow + int remaining = size - readerIdx; + if (remaining < 8) { + streamReader.fillBuffer(8 - remaining); + } + readerIndex = readerIdx + 8; + return Double.longBitsToDouble( + Long.reverseBytes(loadLong(address + readerIdx))); + } + + /** Reads the 1-5 byte int part of a varint. */ + @CodegenInvoke + public int readVarInt32() { + if (AndroidSupport.IS_ANDROID) { + return MemoryOps.readVarInt32(this); + } + if (LITTLE_ENDIAN) { + return _readVarInt32OnLE(); + } else { + return _readVarInt32OnBE(); + } + } + + /** Reads the 1-5 byte as a varint on a little endian machine. */ + @CodegenInvoke + // CHECKSTYLE.OFF:MethodName + public int _readVarInt32OnLE() { + // CHECKSTYLE.ON:MethodName + if (AndroidSupport.IS_ANDROID) { + return MemoryOps.readVarInt32(this); + } + // noinspection Duplicates + int readIdx = readerIndex; + int result; + if (size - readIdx < 5) { + result = readVarUInt32Slow(); + } else { + long address = this.address; + // | 1bit + 7bits | 1bit + 7bits | 1bit + 7bits | 1bit + 7bits | + int fourByteValue = loadInt(address + readIdx); + // Duplicate and manual inline for performance. + // noinspection Duplicates + readIdx++; + result = fourByteValue & 0x7F; + if ((fourByteValue & 0x80) != 0) { + readIdx++; + // 0x3f80: 0b1111111 << 7 + result |= (fourByteValue >>> 1) & 0x3f80; + // 0x8000: 0b1 << 15 + if ((fourByteValue & 0x8000) != 0) { + readIdx++; + // 0x1fc000: 0b1111111 << 14 + result |= (fourByteValue >>> 2) & 0x1fc000; + // 0x800000: 0b1 << 23 + if ((fourByteValue & 0x800000) != 0) { + readIdx++; + // 0xfe00000: 0b1111111 << 21 + result |= (fourByteValue >>> 3) & 0xfe00000; + if ((fourByteValue & 0x80000000) != 0) { + int fifthByte = loadByte(address + readIdx++) & 0xFF; + if ((fifthByte & 0xF0) != 0) { + throwMalformedVarUInt32(fifthByte); + } + result |= fifthByte << 28; + } + } + } + } + readerIndex = readIdx; + } + return (result >>> 1) ^ -(result & 1); + } + + /** Reads the 1-5 byte as a varint on a big endian machine. */ + @CodegenInvoke + // CHECKSTYLE.OFF:MethodName + public int _readVarInt32OnBE() { + // CHECKSTYLE.ON:MethodName + if (AndroidSupport.IS_ANDROID) { + return MemoryOps.readVarInt32(this); + } + // noinspection Duplicates + int readIdx = readerIndex; + int result; + if (size - readIdx < 5) { + result = readVarUInt32Slow(); + } else { + long address = this.address; + int fourByteValue = Integer.reverseBytes(loadInt(address + readIdx)); + // Duplicate and manual inline for performance. + // noinspection Duplicates + readIdx++; + result = fourByteValue & 0x7F; + if ((fourByteValue & 0x80) != 0) { + readIdx++; + // 0x3f80: 0b1111111 << 7 + result |= (fourByteValue >>> 1) & 0x3f80; + // 0x8000: 0b1 << 15 + if ((fourByteValue & 0x8000) != 0) { + readIdx++; + // 0x1fc000: 0b1111111 << 14 + result |= (fourByteValue >>> 2) & 0x1fc000; + // 0x800000: 0b1 << 23 + if ((fourByteValue & 0x800000) != 0) { + readIdx++; + // 0xfe00000: 0b1111111 << 21 + result |= (fourByteValue >>> 3) & 0xfe00000; + if ((fourByteValue & 0x80000000) != 0) { + int fifthByte = loadByte(address + readIdx++) & 0xFF; + if ((fifthByte & 0xF0) != 0) { + throwMalformedVarUInt32(fifthByte); + } + result |= fifthByte << 28; + } + } + } + } + readerIndex = readIdx; + } + return (result >>> 1) ^ -(result & 1); + } + + public long readVarUint36Small() { + if (AndroidSupport.IS_ANDROID) { + return MemoryOps.readVarUint36Small(this); + } + // Android exits above. Keep JVM small-varint bulk reads as raw Unsafe loads instead of calling + // `_unsafeGet*` helpers; those helpers carry Android/endian branches and can break inlining. + // Duplicate and manual inline for performance. + // noinspection Duplicates + int readIdx = readerIndex; + if (size - readIdx >= 9) { + long bulkValue = loadLong(address + readIdx++); + if (!LITTLE_ENDIAN) { + bulkValue = Long.reverseBytes(bulkValue); + } + // noinspection Duplicates + long result = bulkValue & 0x7F; + if ((bulkValue & 0x80) != 0) { + readIdx++; + // 0x3f80: 0b1111111 << 7 + result |= (bulkValue >>> 1) & 0x3f80; + // 0x8000: 0b1 << 15 + if ((bulkValue & 0x8000) != 0) { + return continueReadVarInt36(readIdx, bulkValue, result); + } + } + readerIndex = readIdx; + return result; + } else { + return readVarUint36Slow(); + } + } + + private long continueReadVarInt36(int readIdx, long bulkValue, long result) { + readIdx++; + // 0x1fc000: 0b1111111 << 14 + result |= (bulkValue >>> 2) & 0x1fc000; + // 0x800000: 0b1 << 23 + if ((bulkValue & 0x800000) != 0) { + readIdx++; + // 0xfe00000: 0b1111111 << 21 + result |= (bulkValue >>> 3) & 0xfe00000; + if ((bulkValue & 0x80000000L) != 0) { + readIdx++; + // 0xff0000000: 0b11111111 << 28 + result |= (bulkValue >>> 4) & 0xff0000000L; + } + } + readerIndex = readIdx; + return result; + } + + private long readVarUint36Slow() { + long b = readByte(); + long result = b & 0x7F; + // Note: + // Loop are not used here to improve performance. + // We manually unroll the loop for better performance. + // noinspection Duplicates + if ((b & 0x80) != 0) { + b = readByte(); + result |= (b & 0x7F) << 7; + if ((b & 0x80) != 0) { + b = readByte(); + result |= (b & 0x7F) << 14; + if ((b & 0x80) != 0) { + b = readByte(); + result |= (b & 0x7F) << 21; + if ((b & 0x80) != 0) { + b = readByte(); + result |= (b & 0xFF) << 28; + } + } + } + } + return result; + } + + private int readVarUInt32Slow() { + int b = readByte() & 0xFF; + int result = b & 0x7F; + // Note: + // Loop are not used here to improve performance. + // We manually unroll the loop for better performance. + // noinspection Duplicates + if ((b & 0x80) != 0) { + b = readByte() & 0xFF; + result |= (b & 0x7F) << 7; + if ((b & 0x80) != 0) { + b = readByte() & 0xFF; + result |= (b & 0x7F) << 14; + if ((b & 0x80) != 0) { + b = readByte() & 0xFF; + result |= (b & 0x7F) << 21; + if ((b & 0x80) != 0) { + b = readByte() & 0xFF; + if ((b & 0xF0) != 0) { + throwMalformedVarUInt32(b); + } + result |= b << 28; + } + } + } + } + return result; + } + + private static void throwMalformedVarUInt32(int fifthByte) { + throw new IllegalArgumentException( + "Malformed varuint32 fifth byte " + fifthByte + " exceeds 32 bits"); + } + + /** Reads the 1-5 byte int part of a non-negative varint. */ + public int readVarUInt32() { + if (AndroidSupport.IS_ANDROID) { + return MemoryOps.readVarUInt32(this); + } + int readIdx = readerIndex; + if (size - readIdx < 5) { + return readVarUInt32Slow(); + } + // | 1bit + 7bits | 1bit + 7bits | 1bit + 7bits | 1bit + 7bits | + int fourByteValue = loadInt(address + readIdx); + if (!LITTLE_ENDIAN) { + fourByteValue = Integer.reverseBytes(fourByteValue); + } + readIdx++; + int result = fourByteValue & 0x7F; + // Duplicate and manual inline for performance. + // noinspection Duplicates + if ((fourByteValue & 0x80) != 0) { + readIdx++; + // 0x3f80: 0b1111111 << 7 + result |= (fourByteValue >>> 1) & 0x3f80; + // 0x8000: 0b1 << 15 + if ((fourByteValue & 0x8000) != 0) { + readIdx++; + // 0x1fc000: 0b1111111 << 14 + result |= (fourByteValue >>> 2) & 0x1fc000; + // 0x800000: 0b1 << 23 + if ((fourByteValue & 0x800000) != 0) { + readIdx++; + // 0xfe00000: 0b1111111 << 21 + result |= (fourByteValue >>> 3) & 0xfe00000; + if ((fourByteValue & 0x80000000) != 0) { + int fifthByte = loadByte(address + readIdx++) & 0xFF; + if ((fifthByte & 0xF0) != 0) { + throwMalformedVarUInt32(fifthByte); + } + result |= fifthByte << 28; + } + } + } + } + readerIndex = readIdx; + return result; + } + + /** + * Fast method for read an unsigned varint which is mostly a small value in 7 bits value in [0, + * 127). When the value is equal or greater than 127, the read will be a little slower. + */ + public int readVarUInt32Small7() { + if (AndroidSupport.IS_ANDROID) { + return MemoryOps.readVarUInt32(this); + } + int readIdx = readerIndex; + if (size - readIdx > 0) { + byte v = loadByte(address + readIdx++); + if ((v & 0x80) == 0) { + readerIndex = readIdx; + return v; + } + } + return readVarUInt32Small14(); + } + + /** + * Fast path for read an unsigned varint which is mostly a small value in 14 bits value in [0, + * 16384). When the value is equal or greater than 16384, the read will be a little slower. + */ + public int readVarUInt32Small14() { + if (AndroidSupport.IS_ANDROID) { + return MemoryOps.readVarUInt32(this); + } + int readIdx = readerIndex; + if (size - readIdx >= 5) { + int fourByteValue = loadInt(address + readIdx++); + if (!LITTLE_ENDIAN) { + fourByteValue = Integer.reverseBytes(fourByteValue); + } + int value = fourByteValue & 0x7F; + // Duplicate and manual inline for performance. + // noinspection Duplicates + if ((fourByteValue & 0x80) != 0) { + readIdx++; + value |= (fourByteValue >>> 1) & 0x3f80; + if ((fourByteValue & 0x8000) != 0) { + // merely executed path, make it as a separate method to reduce + // code size of current method for better jvm inline + return continueReadVarUInt32(readIdx, fourByteValue, value); + } + } + readerIndex = readIdx; + return value; + } else { + return readVarUInt32Slow(); + } + } + + private int continueReadVarUInt32(int readIdx, int bulkRead, int value) { + // Duplicate and manual inline for performance. + // noinspection Duplicates + readIdx++; + value |= (bulkRead >>> 2) & 0x1fc000; + if ((bulkRead & 0x800000) != 0) { + readIdx++; + value |= (bulkRead >>> 3) & 0xfe00000; + if ((bulkRead & 0x80000000) != 0) { + int fifthByte = loadByte(address + readIdx++) & 0xFF; + if ((fifthByte & 0xF0) != 0) { + throwMalformedVarUInt32(fifthByte); + } + value |= fifthByte << 28; + } + } + readerIndex = readIdx; + return value; + } + + /** Reads the 1-9 byte int part of a var long. */ + public long readVarInt64() { + if (AndroidSupport.IS_ANDROID) { + return MemoryOps.readVarInt64(this); + } + return LITTLE_ENDIAN ? _readVarInt64OnLE() : _readVarInt64OnBE(); + } + + @CodegenInvoke + // CHECKSTYLE.OFF:MethodName + public long _readVarInt64OnLE() { + // CHECKSTYLE.ON:MethodName + if (AndroidSupport.IS_ANDROID) { + return MemoryOps.readVarInt64(this); + } + // Duplicate and manual inline for performance. + // noinspection Duplicates + int readIdx = readerIndex; + long result; + if (size - readIdx < 9) { + result = readVarUInt64Slow(); + } else { + long address = this.address; + long bulkValue = loadLong(address + readIdx); + // Duplicate and manual inline for performance. + // noinspection Duplicates + readIdx++; + result = bulkValue & 0x7F; + if ((bulkValue & 0x80) != 0) { + readIdx++; + // 0x3f80: 0b1111111 << 7 + result |= (bulkValue >>> 1) & 0x3f80; + // 0x8000: 0b1 << 15 + if ((bulkValue & 0x8000) != 0) { + result = continueReadVarInt64(readIdx, bulkValue, result); + return ((result >>> 1) ^ -(result & 1)); + } + } + readerIndex = readIdx; + } + return ((result >>> 1) ^ -(result & 1)); + } + + @CodegenInvoke + // CHECKSTYLE.OFF:MethodName + public long _readVarInt64OnBE() { + // CHECKSTYLE.ON:MethodName + if (AndroidSupport.IS_ANDROID) { + return MemoryOps.readVarInt64(this); + } + int readIdx = readerIndex; + long result; + if (size - readIdx < 9) { + result = readVarUInt64Slow(); + } else { + long address = this.address; + long bulkValue = Long.reverseBytes(loadLong(address + readIdx)); + // Duplicate and manual inline for performance. + // noinspection Duplicates + readIdx++; + result = bulkValue & 0x7F; + if ((bulkValue & 0x80) != 0) { + readIdx++; + // 0x3f80: 0b1111111 << 7 + result |= (bulkValue >>> 1) & 0x3f80; + // 0x8000: 0b1 << 15 + if ((bulkValue & 0x8000) != 0) { + result = continueReadVarInt64(readIdx, bulkValue, result); + return ((result >>> 1) ^ -(result & 1)); + } + } + readerIndex = readIdx; + } + return ((result >>> 1) ^ -(result & 1)); + } + + /** Reads the 1-9 byte int part of a non-negative var long. */ + public long readVarUInt64() { + if (AndroidSupport.IS_ANDROID) { + return MemoryOps.readVarUInt64(this); + } + int readIdx = readerIndex; + if (size - readIdx < 9) { + return readVarUInt64Slow(); + } + // varint are written using little endian byte order, so read by little endian byte order. + long bulkValue = loadLong(address + readIdx); + if (!LITTLE_ENDIAN) { + bulkValue = Long.reverseBytes(bulkValue); + } + // Duplicate and manual inline for performance. + // noinspection Duplicates + readIdx++; + long result = bulkValue & 0x7F; + if ((bulkValue & 0x80) != 0) { + readIdx++; + // 0x3f80: 0b1111111 << 7 + result |= (bulkValue >>> 1) & 0x3f80; + // 0x8000: 0b1 << 15 + if ((bulkValue & 0x8000) != 0) { + return continueReadVarInt64(readIdx, bulkValue, result); + } + } + readerIndex = readIdx; + return result; + } + + private long continueReadVarInt64(int readIdx, long bulkValue, long result) { + readIdx++; + // 0x1fc000: 0b1111111 << 14 + result |= (bulkValue >>> 2) & 0x1fc000; + // 0x800000: 0b1 << 23 + if ((bulkValue & 0x800000) != 0) { + readIdx++; + // 0xfe00000: 0b1111111 << 21 + result |= (bulkValue >>> 3) & 0xfe00000; + if ((bulkValue & 0x80000000L) != 0) { + readIdx++; + result |= (bulkValue >>> 4) & 0x7f0000000L; + if ((bulkValue & 0x8000000000L) != 0) { + readIdx++; + result |= (bulkValue >>> 5) & 0x3f800000000L; + if ((bulkValue & 0x800000000000L) != 0) { + readIdx++; + result |= (bulkValue >>> 6) & 0x1fc0000000000L; + if ((bulkValue & 0x80000000000000L) != 0) { + readIdx++; + result |= (bulkValue >>> 7) & 0xfe000000000000L; + if ((bulkValue & 0x8000000000000000L) != 0) { + long b = loadByte(address + readIdx++); + result |= b << 56; + } + } + } + } + } + } + readerIndex = readIdx; + return result; + } + + private long readVarUInt64Slow() { + long b = readByte(); + long result = b & 0x7F; + // Note: + // Loop are not used here to improve performance. + // We manually unroll the loop for better performance. + if ((b & 0x80) != 0) { + b = readByte(); + result |= (b & 0x7F) << 7; + if ((b & 0x80) != 0) { + b = readByte(); + result |= (b & 0x7F) << 14; + if ((b & 0x80) != 0) { + b = readByte(); + result |= (b & 0x7F) << 21; + if ((b & 0x80) != 0) { + b = readByte(); + result |= (b & 0x7F) << 28; + if ((b & 0x80) != 0) { + b = readByte(); + result |= (b & 0x7F) << 35; + if ((b & 0x80) != 0) { + b = readByte(); + result |= (b & 0x7F) << 42; + if ((b & 0x80) != 0) { + b = readByte(); + result |= (b & 0x7F) << 49; + if ((b & 0x80) != 0) { + b = readByte(); + // highest bit in last byte is symbols bit. + result |= b << 56; + } + } + } + } + } + } + } + } + return result; + } + + /** Reads the 1-9 byte int part of an aligned varint. */ + public int readAlignedVarUInt32() { + if (AndroidSupport.IS_ANDROID) { + return MemoryOps.readAlignedVarUInt32(this); + } + int readerIdx = readerIndex; + // use subtract to avoid overflow + if (readerIdx < size - 10) { + return slowReadAlignedVarUInt32(); + } + long pos = address + readerIdx; + long startPos = pos; + int b = loadByte(pos++); + // Mask first 6 bits, + // bit 8 `set` indicates have next data bytes. + int result = b & 0x3F; + // Note: + // Loop are not used here to improve performance. + // We manually unroll the loop for better performance. + if ((b & 0x80) != 0) { // has 2nd byte + b = loadByte(pos++); + result |= (b & 0x3F) << 6; + if ((b & 0x80) != 0) { // has 3rd byte + b = loadByte(pos++); + result |= (b & 0x3F) << 12; + if ((b & 0x80) != 0) { // has 4th byte + b = loadByte(pos++); + result |= (b & 0x3F) << 18; + if ((b & 0x80) != 0) { // has 5th byte + b = loadByte(pos++); + result |= (b & 0x3F) << 24; + if ((b & 0x80) != 0) { // has 6th byte + b = loadByte(pos++); + result |= (b & 0x3F) << 30; + } + } + } + } + } + pos = skipPadding(pos, b); // split method for `readVarUint` inlined + readerIndex = (int) (pos - startPos + readerIdx); + return result; + } + + public int slowReadAlignedVarUInt32() { + int b = readByte(); + // Mask first 6 bits, + // bit 8 `set` indicates have next data bytes. + int result = b & 0x3F; + if ((b & 0x80) != 0) { // has 2nd byte + b = readByte(); + result |= (b & 0x3F) << 6; + if ((b & 0x80) != 0) { // has 3rd byte + b = readByte(); + result |= (b & 0x3F) << 12; + if ((b & 0x80) != 0) { // has 4th byte + b = readByte(); + result |= (b & 0x3F) << 18; + if ((b & 0x80) != 0) { // has 5th byte + b = readByte(); + result |= (b & 0x3F) << 24; + if ((b & 0x80) != 0) { // has 6th byte + b = readByte(); + result |= (b & 0x3F) << 30; + } + } + } + } + } + // bit 7 `unset` indicates have next padding bytes, + if ((b & 0x40) == 0) { // has first padding bytes + b = readByte(); + if ((b & 0x40) == 0) { // has 2nd padding bytes + b = readByte(); + if ((b & 0x40) == 0) { // has 3rd padding bytes + b = readByte(); + checkArgument((b & 0x40) != 0, "At most 3 padding bytes."); + } + } + } + return result; + } + + private long skipPadding(long pos, int b) { + // bit 7 `unset` indicates have next padding bytes, + if ((b & 0x40) == 0) { // has first padding bytes + b = loadByte(pos++); + if ((b & 0x40) == 0) { // has 2nd padding bytes + b = loadByte(pos++); + if ((b & 0x40) == 0) { // has 3rd padding bytes + b = loadByte(pos++); + checkArgument((b & 0x40) != 0, "At most 3 padding bytes."); + } + } + } + return pos; + } + + public byte[] readBytes(int length) { + int readerIdx = readerIndex; + byte[] bytes = new byte[length]; + // use subtract to avoid overflow + if (length > size - readerIdx) { + streamReader.readTo(bytes, 0, length); + return bytes; + } + byte[] heapMemory = this.heapMemory; + if (heapMemory != null) { + // System.arraycopy faster for some jdk than Unsafe. + System.arraycopy(heapMemory, heapOffset + readerIdx, bytes, 0, length); + } else { + memoryAccess.copyMemory(null, address + readerIdx, bytes, UnsafeOps.BYTE_ARRAY_OFFSET, length); + } + readerIndex = readerIdx + length; + return bytes; + } + + public void readBytes(byte[] dst, int dstIndex, int length) { + int readerIdx = readerIndex; + // use subtract to avoid overflow + if (readerIdx > size - length) { + streamReader.readTo(dst, dstIndex, length); + return; + } + if (dstIndex < 0 || dstIndex > dst.length - length) { + throwIndexOOBExceptionForRead(); + } + get(readerIdx, dst, dstIndex, length); + readerIndex = readerIdx + length; + } + + public void readBytes(byte[] dst) { + readBytes(dst, 0, dst.length); + } + + /** Read {@code len} bytes into a long using little-endian order. */ + public long readBytesAsInt64(int len) { + if (AndroidSupport.IS_ANDROID) { + return MemoryOps.readBytesAsInt64(this, len); + } + int readerIdx = readerIndex; + // use subtract to avoid overflow + int remaining = size - readerIdx; + if (remaining >= 8) { + readerIndex = readerIdx + len; + long v = loadLong(address + readerIdx); + v = (LITTLE_ENDIAN ? v : Long.reverseBytes(v)) & (0xffffffffffffffffL >>> ((8 - len) * 8)); + return v; + } + return slowReadBytesAsInt64(remaining, len); + } + + private long slowReadBytesAsInt64(int remaining, int len) { + if (remaining < len) { + streamReader.fillBuffer(len - remaining); + } + int readerIdx = readerIndex; + readerIndex = readerIdx + len; + long result = 0; + byte[] heapMemory = this.heapMemory; + if (heapMemory != null) { + for (int i = 0, start = heapOffset + readerIdx; i < len; i++) { + result |= (((long) heapMemory[start + i]) & 0xff) << (i * 8); + } + } else { + long start = address + readerIdx; + for (int i = 0; i < len; i++) { + result |= ((long) memoryAccess.getByte(null, start + i) & 0xff) << (i * 8); + } + } + return result; + } + + public int read(ByteBuffer dst) { + int readerIdx = readerIndex; + int len = dst.remaining(); + // use subtract to avoid overflow + if (readerIdx > size - len) { + return streamReader.readToByteBuffer(dst); + } + if (heapMemory != null) { + dst.put(heapMemory, readerIndex + heapOffset, len); + } else { + dst.put(sliceAsByteBuffer(readerIdx, len)); + } + readerIndex = readerIdx + len; + return len; + } + + public void read(ByteBuffer dst, int len) { + int readerIdx = readerIndex; + // use subtract to avoid overflow + if (readerIdx > size - len) { + streamReader.readToByteBuffer(dst, len); + } else { + if (heapMemory != null) { + dst.put(heapMemory, readerIndex + heapOffset, len); + } else { + dst.put(sliceAsByteBuffer(readerIdx, len)); + } + readerIndex = readerIdx + len; + } + } + + /** + * Read size for following binary, this method will check and fill readable bytes too. This method + * is optimized for small size, it's faster than {@link #readVarUInt32}. + */ + public int readBinarySize() { + if (AndroidSupport.IS_ANDROID) { + return MemoryOps.readBinarySize(this); + } + int binarySize; + int readIdx = readerIndex; + if (size - readIdx >= 5) { + // Android exits above. Keep this small-size fast path as a raw JVM load; `_unsafeGetInt32` + // carries Android/endian branches and can grow the method enough to disturb inlining. + int fourByteValue = loadInt(address + readIdx++); + if (!LITTLE_ENDIAN) { + fourByteValue = Integer.reverseBytes(fourByteValue); + } + binarySize = fourByteValue & 0x7F; + // Duplicate and manual inline for performance. + // noinspection Duplicates + if ((fourByteValue & 0x80) != 0) { + readIdx++; + binarySize |= (fourByteValue >>> 1) & 0x3f80; + if ((fourByteValue & 0x8000) != 0) { + // merely executed path, make it as a separate method to reduce + // code size of current method for better jvm inline + return continueReadBinarySize(readIdx, fourByteValue, binarySize); + } + } + readerIndex = readIdx; + } else { + binarySize = readVarUInt32Slow(); + readIdx = readerIndex; + } + int diff = size - readIdx; + if (diff < binarySize) { + streamReader.fillBuffer(diff); + } + return binarySize; + } + + private int continueReadBinarySize(int readIdx, int bulkRead, int binarySize) { + // Duplicate and manual inline for performance. + // noinspection Duplicates + readIdx++; + binarySize |= (bulkRead >>> 2) & 0x1fc000; + if ((bulkRead & 0x800000) != 0) { + readIdx++; + binarySize |= (bulkRead >>> 3) & 0xfe00000; + if ((bulkRead & 0x80000000) != 0) { + int fifthByte = loadByte(address + readIdx++) & 0xFF; + if ((fifthByte & 0xF0) != 0) { + throwMalformedVarUInt32(fifthByte); + } + binarySize |= fifthByte << 28; + } + } + int diff = size - readIdx; + if (diff < binarySize) { + streamReader.fillBuffer(diff); + } + return binarySize; + } + + public byte[] readBytesAndSize() { + final int numBytes = readBinarySize(); + int readerIdx = readerIndex; + final byte[] arr = new byte[numBytes]; + // use subtract to avoid overflow + if (readerIdx > size - numBytes) { + streamReader.readTo(arr, 0, numBytes); + return arr; + } + byte[] heapMemory = this.heapMemory; + if (heapMemory != null) { + System.arraycopy(heapMemory, heapOffset + readerIdx, arr, 0, numBytes); + } else { + memoryAccess.copyMemory( + null, address + readerIdx, arr, UnsafeOps.BYTE_ARRAY_OFFSET, numBytes); + } + readerIndex = readerIdx + numBytes; + return arr; + } + + /** + * Reads a size-validated primitive byte-array payload into {@code values}. The caller owns size + * validation and destination allocation; this method reads payload bytes only, not the size + * prefix. + */ + public void readByteArrayPayload(byte[] values, int numBytes) { + if (AndroidSupport.IS_ANDROID) { + MemoryOps.readByteArrayPayload(this, values, numBytes); + } else { + int readerIdx = readerIndex; + if (readerIdx > size - numBytes) { + streamReader.readTo(values, 0, numBytes); + return; + } + byte[] heapMemory = this.heapMemory; + if (heapMemory != null) { + System.arraycopy(heapMemory, heapOffset + readerIdx, values, 0, numBytes); + } else { + memoryAccess.copyMemory(null, address + readerIdx, values, UnsafeOps.BYTE_ARRAY_OFFSET, numBytes); + } + readerIndex = readerIdx + numBytes; + } + } + + /** + * Reads a size-validated primitive boolean-array payload into {@code values}. The caller owns + * size validation and destination allocation; this method reads payload bytes only, not the size + * prefix. + */ + public void readBooleanArrayPayload(boolean[] values, int numBytes) { + if (AndroidSupport.IS_ANDROID) { + MemoryOps.readBooleanArrayPayload(this, values, numBytes); + } else { + int readerIdx = readerIndex; + if (readerIdx > size - numBytes) { + streamReader.readBooleans(values, 0, numBytes); + return; + } + memoryAccess.copyMemory( + heapMemory, address + readerIdx, values, UnsafeOps.BOOLEAN_ARRAY_OFFSET, numBytes); + readerIndex = readerIdx + numBytes; + } + } + + /** + * Reads a size-validated primitive char-array payload into {@code values}. The caller owns size + * validation and destination allocation; this method reads payload bytes only, not the size + * prefix. + */ + public void readCharArrayPayload(char[] values, int numBytes) { + if (AndroidSupport.IS_ANDROID) { + MemoryOps.readCharArrayPayload(this, values, numBytes); + } else { + int readerIdx = readerIndex; + if (readerIdx > size - numBytes) { + streamReader.readChars(values, 0, numBytes >>> 1); + return; + } + memoryAccess.copyMemory( + heapMemory, address + readerIdx, values, UnsafeOps.CHAR_ARRAY_OFFSET, numBytes); + readerIndex = readerIdx + numBytes; + } + } + + /** + * Reads a size-validated primitive int16-array payload into {@code values}. The caller owns size + * validation and destination allocation; this method reads payload bytes only, not the size + * prefix. + */ + public void readInt16ArrayPayload(short[] values, int numBytes) { + if (AndroidSupport.IS_ANDROID) { + MemoryOps.readInt16ArrayPayload(this, values, numBytes); + } else { + int readerIdx = readerIndex; + if (readerIdx > size - numBytes) { + streamReader.readShorts(values, 0, numBytes >>> 1); + return; + } + memoryAccess.copyMemory( + heapMemory, address + readerIdx, values, UnsafeOps.SHORT_ARRAY_OFFSET, numBytes); + readerIndex = readerIdx + numBytes; + } + } + + /** + * Reads a size-validated primitive int32-array payload into {@code values}. The caller owns size + * validation and destination allocation; this method reads payload bytes only, not the size + * prefix. + */ + public void readInt32ArrayPayload(int[] values, int numBytes) { + if (AndroidSupport.IS_ANDROID) { + MemoryOps.readInt32ArrayPayload(this, values, numBytes); + } else { + int readerIdx = readerIndex; + if (readerIdx > size - numBytes) { + streamReader.readInts(values, 0, numBytes >>> 2); + return; + } + memoryAccess.copyMemory( + heapMemory, address + readerIdx, values, UnsafeOps.INT_ARRAY_OFFSET, numBytes); + readerIndex = readerIdx + numBytes; + } + } + + /** + * Reads a size-validated primitive int64-array payload into {@code values}. The caller owns size + * validation and destination allocation; this method reads payload bytes only, not the size + * prefix. + */ + public void readInt64ArrayPayload(long[] values, int numBytes) { + if (AndroidSupport.IS_ANDROID) { + MemoryOps.readInt64ArrayPayload(this, values, numBytes); + } else { + int readerIdx = readerIndex; + if (readerIdx > size - numBytes) { + streamReader.readLongs(values, 0, numBytes >>> 3); + return; + } + memoryAccess.copyMemory( + heapMemory, address + readerIdx, values, UnsafeOps.LONG_ARRAY_OFFSET, numBytes); + readerIndex = readerIdx + numBytes; + } + } + + /** + * Reads a size-validated primitive float32-array payload into {@code values}. The caller owns + * size validation and destination allocation; this method reads payload bytes only, not the size + * prefix. + */ + public void readFloat32ArrayPayload(float[] values, int numBytes) { + if (AndroidSupport.IS_ANDROID) { + MemoryOps.readFloat32ArrayPayload(this, values, numBytes); + } else { + int readerIdx = readerIndex; + if (readerIdx > size - numBytes) { + streamReader.readFloats(values, 0, numBytes >>> 2); + return; + } + memoryAccess.copyMemory( + heapMemory, address + readerIdx, values, UnsafeOps.FLOAT_ARRAY_OFFSET, numBytes); + readerIndex = readerIdx + numBytes; + } + } + + /** + * Reads a size-validated primitive float64-array payload into {@code values}. The caller owns + * size validation and destination allocation; this method reads payload bytes only, not the size + * prefix. + */ + public void readFloat64ArrayPayload(double[] values, int numBytes) { + if (AndroidSupport.IS_ANDROID) { + MemoryOps.readFloat64ArrayPayload(this, values, numBytes); + } else { + int readerIdx = readerIndex; + if (readerIdx > size - numBytes) { + streamReader.readDoubles(values, 0, numBytes >>> 3); + return; + } + memoryAccess.copyMemory( + heapMemory, address + readerIdx, values, UnsafeOps.DOUBLE_ARRAY_OFFSET, numBytes); + readerIndex = readerIdx + numBytes; + } + } + + public void readBooleans(boolean[] values, int offset, int numElements) { + if (AndroidSupport.IS_ANDROID) { + MemoryOps.readBooleans(this, values, offset, numElements); + } else { + if ((offset | numElements | (offset + numElements) | (values.length - offset - numElements)) + < 0) { + throwOOBException(); + } + if (readerIndex > size - numElements) { + streamReader.readBooleans(values, offset, numElements); + return; + } + int readerIdx = readerIndex; + memoryAccess.copyMemory( + heapMemory, + address + readerIdx, + values, + UnsafeOps.BOOLEAN_ARRAY_OFFSET + offset, + numElements); + readerIndex = readerIdx + numElements; + } + } + + public void readChars(char[] chars, int numElements) { + readChars(chars, 0, numElements); + } + + public void readChars(char[] chars, int offset, int numElements) { + if (AndroidSupport.IS_ANDROID) { + MemoryOps.readChars(this, chars, offset, numElements); + } else { + if ((offset | numElements | (offset + numElements) | (chars.length - offset - numElements)) + < 0) { + throwOOBException(); + } + int numBytes = Math.multiplyExact(numElements, 2); + if (readerIndex > size - numBytes) { + streamReader.readChars(chars, offset, numElements); + return; + } + int readerIdx = readerIndex; + memoryAccess.copyMemory( + heapMemory, + address + readerIdx, + chars, + UnsafeOps.CHAR_ARRAY_OFFSET + ((long) offset << 1), + numBytes); + readerIndex = readerIdx + numBytes; + } + } + + @CodegenInvoke + public char[] readCharsAndSize() { + final int numBytes = readBinarySize(); + int numElements = numBytes >> 1; + char[] values = new char[numElements]; + readChars(values, 0, numElements); + return values; + } + + public void readShorts(short[] values, int offset, int numElements) { + if (AndroidSupport.IS_ANDROID) { + MemoryOps.readShorts(this, values, offset, numElements); + } else { + if ((offset | numElements | (offset + numElements) | (values.length - offset - numElements)) + < 0) { + throwOOBException(); + } + int numBytes = Math.multiplyExact(numElements, 2); + if (readerIndex > size - numBytes) { + streamReader.readShorts(values, offset, numElements); + return; + } + int readerIdx = readerIndex; + memoryAccess.copyMemory( + heapMemory, + address + readerIdx, + values, + UnsafeOps.SHORT_ARRAY_OFFSET + ((long) offset << 1), + numBytes); + readerIndex = readerIdx + numBytes; + } + } + + public void readInts(int[] values, int offset, int numElements) { + if (AndroidSupport.IS_ANDROID) { + MemoryOps.readInts(this, values, offset, numElements); + } else { + if ((offset | numElements | (offset + numElements) | (values.length - offset - numElements)) + < 0) { + throwOOBException(); + } + int numBytes = Math.multiplyExact(numElements, 4); + if (readerIndex > size - numBytes) { + streamReader.readInts(values, offset, numElements); + return; + } + int readerIdx = readerIndex; + memoryAccess.copyMemory( + heapMemory, + address + readerIdx, + values, + UnsafeOps.INT_ARRAY_OFFSET + ((long) offset << 2), + numBytes); + readerIndex = readerIdx + numBytes; + } + } + + public void readLongs(long[] values, int offset, int numElements) { + if (AndroidSupport.IS_ANDROID) { + MemoryOps.readLongs(this, values, offset, numElements); + } else { + if ((offset | numElements | (offset + numElements) | (values.length - offset - numElements)) + < 0) { + throwOOBException(); + } + int numBytes = Math.multiplyExact(numElements, 8); + if (readerIndex > size - numBytes) { + streamReader.readLongs(values, offset, numElements); + return; + } + int readerIdx = readerIndex; + memoryAccess.copyMemory( + heapMemory, + address + readerIdx, + values, + UnsafeOps.LONG_ARRAY_OFFSET + ((long) offset << 3), + numBytes); + readerIndex = readerIdx + numBytes; + } + } + + public void readFloats(float[] values, int offset, int numElements) { + if (AndroidSupport.IS_ANDROID) { + MemoryOps.readFloats(this, values, offset, numElements); + } else { + if ((offset | numElements | (offset + numElements) | (values.length - offset - numElements)) + < 0) { + throwOOBException(); + } + int numBytes = Math.multiplyExact(numElements, 4); + if (readerIndex > size - numBytes) { + streamReader.readFloats(values, offset, numElements); + return; + } + int readerIdx = readerIndex; + memoryAccess.copyMemory( + heapMemory, + address + readerIdx, + values, + UnsafeOps.FLOAT_ARRAY_OFFSET + ((long) offset << 2), + numBytes); + readerIndex = readerIdx + numBytes; + } + } + + public void readDoubles(double[] values, int offset, int numElements) { + if (AndroidSupport.IS_ANDROID) { + MemoryOps.readDoubles(this, values, offset, numElements); + } else { + if ((offset | numElements | (offset + numElements) | (values.length - offset - numElements)) + < 0) { + throwOOBException(); + } + int numBytes = Math.multiplyExact(numElements, 8); + if (readerIndex > size - numBytes) { + streamReader.readDoubles(values, offset, numElements); + return; + } + int readerIdx = readerIndex; + memoryAccess.copyMemory( + heapMemory, + address + readerIdx, + values, + UnsafeOps.DOUBLE_ARRAY_OFFSET + ((long) offset << 3), + numBytes); + readerIndex = readerIdx + numBytes; + } + } + + public void checkReadableBytes(int minimumReadableBytes) { + // use subtract to avoid overflow + int remaining = size - readerIndex; + if (minimumReadableBytes > remaining) { + streamReader.fillBuffer(minimumReadableBytes - remaining); + } + } + + /** + * Returns internal byte array if data is on heap and remaining buffer size is equal to internal + * byte array size, or create a new byte array which copy remaining data from off-heap. + */ + public byte[] getRemainingBytes() { + int length = size - readerIndex; + if (heapMemory != null && size == length && heapOffset == 0) { + return heapMemory; + } else { + return getBytes(readerIndex, length); + } + } + + // ------------------------- Read Methods Finished ------------------------------------- + + public void copyTo(int offset, MemoryBuffer target, int targetOffset, int numBytes) { + if (AndroidSupport.IS_ANDROID) { + MemoryOps.copyTo(this, offset, target, targetOffset, numBytes); + } else { + final byte[] thisHeapRef = this.heapMemory; + final byte[] otherHeapRef = target.heapMemory; + final long thisPointer = this.address + offset; + final long otherPointer = target.address + targetOffset; + if ((numBytes | offset | targetOffset) >= 0 + && thisPointer <= this.addressLimit - numBytes + && otherPointer <= target.addressLimit - numBytes) { + if (thisHeapRef != null && otherHeapRef != null) { + memoryAccess.copyMemory(thisHeapRef, thisPointer, otherHeapRef, otherPointer, numBytes); + } else if (sameBufferOverlap(target, offset, targetOffset, numBytes)) { + byte[] tmp = new byte[numBytes]; + sliceAsByteBuffer(offset, numBytes).get(tmp); + target.sliceAsByteBuffer(targetOffset, numBytes).put(tmp); + } else { + target.sliceAsByteBuffer(targetOffset, numBytes).put(sliceAsByteBuffer(offset, numBytes)); + } + } else { + throw new IndexOutOfBoundsException( + String.format( + "offset=%d, targetOffset=%d, numBytes=%d, address=%d, targetAddress=%d", + offset, targetOffset, numBytes, this.address, target.address)); + } + } + } + + private boolean sameBufferOverlap( + MemoryBuffer target, int offset, int targetOffset, int numBytes) { + if (numBytes <= 0) { + return false; + } + if (heapMemory != null && heapMemory == target.heapMemory) { + int sourceStart = heapOffset + offset; + int targetStart = target.heapOffset + targetOffset; + return sourceStart < targetStart + numBytes && targetStart < sourceStart + numBytes; + } + if (offHeapBuffer != null && offHeapBuffer == target.offHeapBuffer) { + long sourceStart = address + offset; + long targetStart = target.address + targetOffset; + return sourceStart < targetStart + numBytes && targetStart < sourceStart + numBytes; + } + return false; + } + + public void copyFrom(int offset, MemoryBuffer source, int sourcePointer, int numBytes) { + source.copyTo(sourcePointer, this, offset, numBytes); + } + + /** + * JVM-only bulk copy method. Copies {@code numBytes} bytes to target unsafe object and pointer. + * Throws on Android before executing unsafe memory access. + */ + public void copyToUnsafe(long offset, Object target, long targetPointer, int numBytes) { + if (AndroidSupport.IS_ANDROID) { + MemoryOps.throwRawUnsafeMemoryCopyUnsupported(); + } else { + checkArgument(target != null, "Raw native-address target copy is unsupported on JDK25"); + final long thisPointer = this.address + offset; + checkArgument(thisPointer + numBytes <= addressLimit); + memoryAccess.copyMemory(this.heapMemory, thisPointer, target, targetPointer, numBytes); + } + } + + /** + * JVM-only bulk copy method. Copies {@code numBytes} bytes from source unsafe object and pointer. + * Throws on Android before executing unsafe memory access. + */ + public void copyFromUnsafe(long offset, Object source, long sourcePointer, long numBytes) { + if (AndroidSupport.IS_ANDROID) { + MemoryOps.throwRawUnsafeMemoryCopyUnsupported(); + } else { + checkArgument(source != null, "Raw native-address source copy is unsupported on JDK25"); + final long thisPointer = this.address + offset; + checkArgument(thisPointer + numBytes <= addressLimit); + memoryAccess.copyMemory(source, sourcePointer, heapMemory, thisPointer, numBytes); + } + } + + public byte[] getBytes(int index, int length) { + if (index == 0 && heapMemory != null && heapOffset == 0) { + // Arrays.copyOf is an intrinsics, which is faster + return Arrays.copyOf(heapMemory, length); + } + if (index + length > size) { + throwIndexOOBExceptionForRead(length); + } + byte[] data = new byte[length]; + get(index, data, 0, length); + return data; + } + + public void getBytes(int index, byte[] dst, int dstIndex, int length) { + if (dstIndex > dst.length - length) { + throwOOBException(); + } + if (index > size - length) { + throwOOBException(); + } + get(index, dst, dstIndex, length); + } + + public MemoryBuffer slice(int offset) { + return slice(offset, size - offset); + } + + public MemoryBuffer slice(int offset, int length) { + if (offset + length > size) { + throwOOBExceptionForRange(offset, length); + } + if (heapMemory != null) { + return new MemoryBuffer(heapMemory, heapOffset + offset, length); + } else { + return new MemoryBuffer(address + offset, length, offHeapBuffer); + } + } + + public ByteBuffer sliceAsByteBuffer() { + return sliceAsByteBuffer(readerIndex, size - readerIndex); + } + + public ByteBuffer sliceAsByteBuffer(int offset, int length) { + if (offset + length > size) { + throwOOBExceptionForRange(offset, length); + } + if (heapMemory != null) { + return ByteBuffer.wrap(heapMemory, heapOffset + offset, length).slice(); + } else { + ByteBuffer offHeapBuffer = this.offHeapBuffer; + if (offHeapBuffer != null) { + ByteBuffer duplicate = offHeapBuffer.duplicate(); + int start = (int) address; + ByteBufferUtil.position(duplicate, start + offset); + duplicate.limit(start + offset + length); + return duplicate.slice(); + } else { + throw new IllegalStateException("Memory buffer does not own a ByteBuffer"); + } + } + } + + private void throwOOBExceptionForRange(int offset, int length) { + throw new IndexOutOfBoundsException( + String.format("offset(%d) + length(%d) exceeds size(%d): %s", offset, length, size, this)); + } + + public ForyStreamReader getStreamReader() { + return streamReader; + } + + /** + * Equals two memory buffer regions. + * + * @param buf2 Buffer to equal this buffer with + * @param offset1 Offset of this buffer to start equaling + * @param offset2 Offset of buf2 to start equaling + * @param len Length of the equaled memory region + * @return true if buffers equal or len zero, false otherwise + */ + public boolean equalTo(MemoryBuffer buf2, int offset1, int offset2, int len) { + if (len == 0) { + return buf2 != null; + } + if (AndroidSupport.IS_ANDROID) { + return MemoryOps.equalTo(this, buf2, offset1, offset2, len); + } + final long pos1 = address + offset1; + final long pos2 = buf2.address + offset2; + checkArgument(pos1 < addressLimit); + checkArgument(pos2 < buf2.addressLimit); + return unsafeEqualTo(memoryAccess, heapMemory, pos1, buf2.memoryAccess, buf2.heapMemory, pos2, len); + } + + /** + * Equals a memory buffer region with a byte array region. + * + * @param bytes Array to compare with + * @param bytesOffset Offset of bytes to start comparing + * @param offset Offset of this buffer to start comparing + * @param len Length of the compared memory region + * @return true if regions are equal or len zero, false otherwise + */ + public boolean equalTo(byte[] bytes, int bytesOffset, int offset, int len) { + checkArgument(bytes != null); + checkArgument(len >= 0); + checkArgument(bytesOffset >= 0 && bytesOffset <= bytes.length - len); + checkArgument(offset >= 0 && offset <= size - len); + if (len == 0) { + return true; + } + if (AndroidSupport.IS_ANDROID) { + return MemoryOps.equalTo(this, bytes, bytesOffset, offset, len); + } + final long pos = address + offset; + return unsafeEqualTo( + memoryAccess, heapMemory, pos, memoryAccess, bytes, UnsafeOps.BYTE_ARRAY_OFFSET + bytesOffset, len); + } + + private static boolean unsafeEqualTo( + MemoryAccess leftAccess, + Object leftBase, + long leftOffset, + MemoryAccess rightAccess, + Object rightBase, + long rightOffset, + int length) { + int i = 0; + if ((leftOffset % 8) == (rightOffset % 8)) { + while ((leftOffset + i) % 8 != 0 && i < length) { + if (leftAccess.getByte(leftBase, leftOffset + i) + != rightAccess.getByte(rightBase, rightOffset + i)) { + return false; + } + i += 1; + } + } + if (UNALIGNED || (((leftOffset + i) % 8 == 0) && ((rightOffset + i) % 8 == 0))) { + while (i <= length - 8) { + if (leftAccess.getLong(leftBase, leftOffset + i) + != rightAccess.getLong(rightBase, rightOffset + i)) { + return false; + } + i += 8; + } + } + while (i < length) { + if (leftAccess.getByte(leftBase, leftOffset + i) + != rightAccess.getByte(rightBase, rightOffset + i)) { + return false; + } + i += 1; + } + return true; + } + + private static final class MemoryAccess { + private final MemoryBuffer buffer; + + private MemoryAccess(MemoryBuffer buffer) { + this.buffer = buffer; + } + + private byte getByte(Object base, long offset) { + if (base != null) { + return UnsafeOps.getByte(base, offset); + } + return directSegment().get(ValueLayout.JAVA_BYTE, offset); + } + + private void putByte(Object base, long offset, byte value) { + if (base != null) { + UnsafeOps.putByte(base, offset, value); + return; + } + directSegment().set(ValueLayout.JAVA_BYTE, offset, value); + } + + private char getChar(Object base, long offset) { + if (base != null) { + return UnsafeOps.getChar(base, offset); + } + return directSegment().get(NATIVE_CHAR, offset); + } + + private void putChar(Object base, long offset, char value) { + if (base != null) { + UnsafeOps.putChar(base, offset, value); + return; + } + directSegment().set(NATIVE_CHAR, offset, value); + } + + private short getShort(Object base, long offset) { + if (base != null) { + return UnsafeOps.getShort(base, offset); + } + return directSegment().get(NATIVE_SHORT, offset); + } + + private void putShort(Object base, long offset, short value) { + if (base != null) { + UnsafeOps.putShort(base, offset, value); + return; + } + directSegment().set(NATIVE_SHORT, offset, value); + } + + private int getInt(Object base, long offset) { + if (base != null) { + return UnsafeOps.getInt(base, offset); + } + return directSegment().get(NATIVE_INT, offset); + } + + private void putInt(Object base, long offset, int value) { + if (base != null) { + UnsafeOps.putInt(base, offset, value); + return; + } + directSegment().set(NATIVE_INT, offset, value); + } + + private long getLong(Object base, long offset) { + if (base != null) { + return UnsafeOps.getLong(base, offset); + } + return directSegment().get(NATIVE_LONG, offset); + } + + private void putLong(Object base, long offset, long value) { + if (base != null) { + UnsafeOps.putLong(base, offset, value); + return; + } + directSegment().set(NATIVE_LONG, offset, value); + } + + private void copyMemory( + Object src, long srcOffset, Object dst, long dstOffset, long length) { + int len = toIntLength(length); + if (len == 0) { + return; + } + if (src != null && dst != null) { + if (dst instanceof byte[] && copyArrayToBytes(src, srcOffset, (byte[]) dst, toIntIndex(dstOffset), len)) { + return; + } + if (src instanceof byte[] && copyBytesToArray((byte[]) src, toIntIndex(srcOffset), dst, dstOffset, len)) { + return; + } + UnsafeOps.copyMemory(src, srcOffset, dst, dstOffset, len); + } else if (src == null && dst == null) { + copyDirect(srcOffset, dstOffset, len); + } else if (src == null) { + if (dst instanceof byte[]) { + readDirect(srcOffset, (byte[]) dst, toIntIndex(dstOffset), len); + } else if (readArray(srcOffset, dst, dstOffset, len)) { + return; + } else { + for (int i = 0; i < len; i++) { + UnsafeOps.putByte(dst, dstOffset + i, getByte(null, srcOffset + i)); + } + } + } else if (src instanceof byte[]) { + writeDirect(dstOffset, (byte[]) src, toIntIndex(srcOffset), len); + } else if (writeArray(src, srcOffset, dstOffset, len)) { + return; + } else { + for (int i = 0; i < len; i++) { + putByte(null, dstOffset + i, UnsafeOps.getByte(src, srcOffset + i)); + } + } + } + + private boolean copyArrayToBytes( + Object src, long srcOffset, byte[] dst, int dstOffset, int len) { + if (src instanceof boolean[]) { + boolean[] array = (boolean[]) src; + int srcIndex = toIntIndex(srcOffset); + for (int i = 0; i < len; i++) { + dst[dstOffset + i] = array[srcIndex + i] ? (byte) 1 : (byte) 0; + } + return true; + } else if (src instanceof char[] && aligned(srcOffset, len, Character.BYTES)) { + heapBytes(dst, dstOffset, len) + .asCharBuffer() + .put((char[]) src, toIntIndex(srcOffset / Character.BYTES), len / Character.BYTES); + return true; + } else if (src instanceof short[] && aligned(srcOffset, len, Short.BYTES)) { + heapBytes(dst, dstOffset, len) + .asShortBuffer() + .put((short[]) src, toIntIndex(srcOffset / Short.BYTES), len / Short.BYTES); + return true; + } else if (src instanceof int[] && aligned(srcOffset, len, Integer.BYTES)) { + heapBytes(dst, dstOffset, len) + .asIntBuffer() + .put((int[]) src, toIntIndex(srcOffset / Integer.BYTES), len / Integer.BYTES); + return true; + } else if (src instanceof long[] && aligned(srcOffset, len, Long.BYTES)) { + heapBytes(dst, dstOffset, len) + .asLongBuffer() + .put((long[]) src, toIntIndex(srcOffset / Long.BYTES), len / Long.BYTES); + return true; + } else if (src instanceof float[] && aligned(srcOffset, len, Float.BYTES)) { + heapBytes(dst, dstOffset, len) + .asFloatBuffer() + .put((float[]) src, toIntIndex(srcOffset / Float.BYTES), len / Float.BYTES); + return true; + } else if (src instanceof double[] && aligned(srcOffset, len, Double.BYTES)) { + heapBytes(dst, dstOffset, len) + .asDoubleBuffer() + .put((double[]) src, toIntIndex(srcOffset / Double.BYTES), len / Double.BYTES); + return true; + } + return false; + } + + private boolean copyBytesToArray( + byte[] src, int srcOffset, Object dst, long dstOffset, int len) { + if (dst instanceof boolean[]) { + boolean[] array = (boolean[]) dst; + int dstIndex = toIntIndex(dstOffset); + for (int i = 0; i < len; i++) { + array[dstIndex + i] = src[srcOffset + i] != 0; + } + return true; + } else if (dst instanceof char[] && aligned(dstOffset, len, Character.BYTES)) { + heapBytes(src, srcOffset, len) + .asCharBuffer() + .get((char[]) dst, toIntIndex(dstOffset / Character.BYTES), len / Character.BYTES); + return true; + } else if (dst instanceof short[] && aligned(dstOffset, len, Short.BYTES)) { + heapBytes(src, srcOffset, len) + .asShortBuffer() + .get((short[]) dst, toIntIndex(dstOffset / Short.BYTES), len / Short.BYTES); + return true; + } else if (dst instanceof int[] && aligned(dstOffset, len, Integer.BYTES)) { + heapBytes(src, srcOffset, len) + .asIntBuffer() + .get((int[]) dst, toIntIndex(dstOffset / Integer.BYTES), len / Integer.BYTES); + return true; + } else if (dst instanceof long[] && aligned(dstOffset, len, Long.BYTES)) { + heapBytes(src, srcOffset, len) + .asLongBuffer() + .get((long[]) dst, toIntIndex(dstOffset / Long.BYTES), len / Long.BYTES); + return true; + } else if (dst instanceof float[] && aligned(dstOffset, len, Float.BYTES)) { + heapBytes(src, srcOffset, len) + .asFloatBuffer() + .get((float[]) dst, toIntIndex(dstOffset / Float.BYTES), len / Float.BYTES); + return true; + } else if (dst instanceof double[] && aligned(dstOffset, len, Double.BYTES)) { + heapBytes(src, srcOffset, len) + .asDoubleBuffer() + .get((double[]) dst, toIntIndex(dstOffset / Double.BYTES), len / Double.BYTES); + return true; + } + return false; + } + + private boolean writeArray(Object src, long srcOffset, long dstOffset, int len) { + if (src instanceof boolean[]) { + boolean[] array = (boolean[]) src; + int srcIndex = toIntIndex(srcOffset); + ByteBuffer dst = directBytes(dstOffset, len); + for (int i = 0; i < len; i++) { + dst.put(i, array[srcIndex + i] ? (byte) 1 : (byte) 0); + } + return true; + } else if (src instanceof char[] && aligned(srcOffset, len, Character.BYTES)) { + directBytes(dstOffset, len) + .asCharBuffer() + .put((char[]) src, toIntIndex(srcOffset / Character.BYTES), len / Character.BYTES); + return true; + } else if (src instanceof short[] && aligned(srcOffset, len, Short.BYTES)) { + directBytes(dstOffset, len) + .asShortBuffer() + .put((short[]) src, toIntIndex(srcOffset / Short.BYTES), len / Short.BYTES); + return true; + } else if (src instanceof int[] && aligned(srcOffset, len, Integer.BYTES)) { + directBytes(dstOffset, len) + .asIntBuffer() + .put((int[]) src, toIntIndex(srcOffset / Integer.BYTES), len / Integer.BYTES); + return true; + } else if (src instanceof long[] && aligned(srcOffset, len, Long.BYTES)) { + directBytes(dstOffset, len) + .asLongBuffer() + .put((long[]) src, toIntIndex(srcOffset / Long.BYTES), len / Long.BYTES); + return true; + } else if (src instanceof float[] && aligned(srcOffset, len, Float.BYTES)) { + directBytes(dstOffset, len) + .asFloatBuffer() + .put((float[]) src, toIntIndex(srcOffset / Float.BYTES), len / Float.BYTES); + return true; + } else if (src instanceof double[] && aligned(srcOffset, len, Double.BYTES)) { + directBytes(dstOffset, len) + .asDoubleBuffer() + .put((double[]) src, toIntIndex(srcOffset / Double.BYTES), len / Double.BYTES); + return true; + } + return false; + } + + private boolean readArray(long srcOffset, Object dst, long dstOffset, int len) { + if (dst instanceof boolean[]) { + boolean[] array = (boolean[]) dst; + int dstIndex = toIntIndex(dstOffset); + ByteBuffer src = directBytes(srcOffset, len); + for (int i = 0; i < len; i++) { + array[dstIndex + i] = src.get(i) != 0; + } + return true; + } else if (dst instanceof char[] && aligned(dstOffset, len, Character.BYTES)) { + directBytes(srcOffset, len) + .asCharBuffer() + .get((char[]) dst, toIntIndex(dstOffset / Character.BYTES), len / Character.BYTES); + return true; + } else if (dst instanceof short[] && aligned(dstOffset, len, Short.BYTES)) { + directBytes(srcOffset, len) + .asShortBuffer() + .get((short[]) dst, toIntIndex(dstOffset / Short.BYTES), len / Short.BYTES); + return true; + } else if (dst instanceof int[] && aligned(dstOffset, len, Integer.BYTES)) { + directBytes(srcOffset, len) + .asIntBuffer() + .get((int[]) dst, toIntIndex(dstOffset / Integer.BYTES), len / Integer.BYTES); + return true; + } else if (dst instanceof long[] && aligned(dstOffset, len, Long.BYTES)) { + directBytes(srcOffset, len) + .asLongBuffer() + .get((long[]) dst, toIntIndex(dstOffset / Long.BYTES), len / Long.BYTES); + return true; + } else if (dst instanceof float[] && aligned(dstOffset, len, Float.BYTES)) { + directBytes(srcOffset, len) + .asFloatBuffer() + .get((float[]) dst, toIntIndex(dstOffset / Float.BYTES), len / Float.BYTES); + return true; + } else if (dst instanceof double[] && aligned(dstOffset, len, Double.BYTES)) { + directBytes(srcOffset, len) + .asDoubleBuffer() + .get((double[]) dst, toIntIndex(dstOffset / Double.BYTES), len / Double.BYTES); + return true; + } + return false; + } + + private void copyDirect(long srcOffset, long dstOffset, int len) { + if (srcOffset < dstOffset && dstOffset < srcOffset + len) { + byte[] tmp = new byte[len]; + readDirect(srcOffset, tmp, 0, len); + writeDirect(dstOffset, tmp, 0, len); + } else { + MemorySegment segment = directSegment(); + MemorySegment.copy(segment, srcOffset, segment, dstOffset, len); + } + } + + private ByteBuffer directBuffer() { + ByteBuffer directBuffer = buffer.nativeOffHeapBuffer; + if (directBuffer == null) { + throw new IllegalStateException("Memory buffer does not own a ByteBuffer"); + } + return directBuffer; + } + + private MemorySegment directSegment() { + MemorySegment segment = buffer.offHeapSegment; + if (segment == null) { + throw new IllegalStateException("Memory buffer does not own an off-heap segment"); + } + return segment; + } + + private void readDirect(long offset, byte[] dst, int dstOffset, int length) { + directBuffer().get(toIntIndex(offset), dst, dstOffset, length); + } + + private void writeDirect(long offset, byte[] src, int srcOffset, int length) { + directBuffer().put(toIntIndex(offset), src, srcOffset, length); + } + + private ByteBuffer directBytes(long offset, int length) { + ByteBuffer duplicate = directBuffer().duplicate().order(NATIVE_ORDER); + int start = toIntIndex(offset); + ByteBufferUtil.position(duplicate, start); + duplicate.limit(start + length); + return duplicate.slice().order(NATIVE_ORDER); + } + + private static ByteBuffer heapBytes(byte[] bytes, int offset, int length) { + return ByteBuffer.wrap(bytes, offset, length).order(NATIVE_ORDER); + } + + private static boolean aligned(long offset, int length, int width) { + return offset % width == 0 && length % width == 0; + } + + private static int toIntIndex(long offset) { + if (offset < 0 || offset > Integer.MAX_VALUE) { + throw new IndexOutOfBoundsException("offset out of int range: " + offset); + } + return (int) offset; + } + + private static int toIntLength(long length) { + if (length < 0 || length > Integer.MAX_VALUE) { + throw new IndexOutOfBoundsException("length out of int range: " + length); + } + return (int) length; + } + } + + @Override + public String toString() { + return "MemoryBuffer{" + + "size=" + + size + + ", readerIndex=" + + readerIndex + + ", writerIndex=" + + writerIndex + + ", heapMemory=" + + (heapMemory == null ? null : "len(" + heapMemory.length + ")") + + ", heapOffset=" + + heapOffset + + ", offHeapBuffer=" + + offHeapBuffer + + ", address=" + + address + + ", addressLimit=" + + addressLimit + + '}'; + } + + // ------------------------------------------------------------------------ + // Memory Allocator Support + // ------------------------------------------------------------------------ + + /** Default memory allocator that uses the original heap-based allocation strategy. */ + private static final class DefaultMemoryAllocator implements MemoryAllocator { + @Override + public MemoryBuffer allocate(int initialSize) { + return fromByteArray(new byte[initialSize]); + } + + @Override + public void grow(MemoryBuffer buffer, int newCapacity) { + if (newCapacity <= buffer.size()) { + return; + } + + int newSize = + newCapacity < BUFFER_GROW_STEP_THRESHOLD + ? newCapacity << 1 + : (int) Math.min(newCapacity * 1.5d, Integer.MAX_VALUE - 8); + + byte[] data = new byte[newSize]; + buffer.get(0, data, 0, buffer.size()); + buffer.initHeapBuffer(data, 0, data.length); + } + } + + /** + * Sets the global memory allocator. This affects all new MemoryBuffer allocations and growth + * operations. + * + * @param allocator the new global allocator to use + * @throws NullPointerException if allocator is null + */ + public static void setGlobalAllocator(MemoryAllocator allocator) { + if (allocator == null) { + throw new NullPointerException("Memory allocator cannot be null"); + } + globalAllocator = allocator; + } + + /** + * Gets the current global memory allocator. + * + * @return the current global allocator + */ + public static MemoryAllocator getGlobalAllocator() { + return globalAllocator; + } + + /** Point this buffer to a new byte array. */ + public void pointTo(byte[] buffer, int offset, int length) { + initHeapBuffer(buffer, offset, length); + } + + /** Creates a new memory buffer that targets to the given heap memory region. */ + public static MemoryBuffer fromByteArray(byte[] buffer, int offset, int length) { + return new MemoryBuffer(buffer, offset, length, null); + } + + public static MemoryBuffer fromByteArray( + byte[] buffer, int offset, int length, ForyStreamReader streamReader) { + return new MemoryBuffer(buffer, offset, length, streamReader); + } + + /** Creates a new memory buffer that targets to the given heap memory region. */ + public static MemoryBuffer fromByteArray(byte[] buffer) { + return new MemoryBuffer(buffer, 0, buffer.length); + } + + /** + * Creates a new memory buffer that represents the memory backing the given byte buffer section of + * {@code [buffer.position(), buffer.limit())}. The buffer will change into a heap buffer + * automatically if not enough. + * + * @param buffer a direct buffer or heap buffer + */ + public static MemoryBuffer fromByteBuffer(ByteBuffer buffer) { + if (AndroidSupport.IS_ANDROID) { + return MemoryOps.fromByteBuffer(buffer); + } else if (buffer.isDirect()) { + return new MemoryBuffer(buffer.position(), buffer.remaining(), buffer); + } else if (buffer.hasArray()) { + int offset = buffer.arrayOffset() + buffer.position(); + return new MemoryBuffer(buffer.array(), offset, buffer.remaining()); + } else { + ByteBuffer duplicate = buffer.duplicate(); + byte[] bytes = new byte[duplicate.remaining()]; + duplicate.get(bytes); + return fromByteArray(bytes); + } + } + + public static MemoryBuffer fromDirectByteBuffer( + ByteBuffer buffer, int size, ForyStreamReader streamReader) { + if (AndroidSupport.IS_ANDROID) { + return MemoryOps.directByteBufferUnsupported(); + } + long offHeapAddress = buffer.position(); + return new MemoryBuffer(offHeapAddress, size, buffer, streamReader); + } + + /** + * Create a heap buffer of specified initial size. The buffer will grow automatically if not + * enough. + */ + public static MemoryBuffer newHeapBuffer(int initialSize) { + return globalAllocator.allocate(initialSize); + } +} diff --git a/java/fory-core/src/main/java25/org/apache/fory/platform/UnsafeOps.java b/java/fory-core/src/main/java25/org/apache/fory/platform/UnsafeOps.java index ce00b9b6c7..cd478704d3 100644 --- a/java/fory-core/src/main/java25/org/apache/fory/platform/UnsafeOps.java +++ b/java/fory-core/src/main/java25/org/apache/fory/platform/UnsafeOps.java @@ -34,6 +34,9 @@ public final class UnsafeOps { @SuppressWarnings("restriction") public static final Unsafe UNSAFE = _JDKAccess.UNSAFE; + // JDK25 array operations use Java/VarHandle indexes instead of raw Unsafe byte offsets. + // Keep these constants zero so versioned MemoryBuffer code can preserve the root API shape + // without mixing Unsafe base-offset domains into the zero-Unsafe runtime. public static final int BOOLEAN_ARRAY_OFFSET = 0; public static final int BYTE_ARRAY_OFFSET = 0; public static final int CHAR_ARRAY_OFFSET = 0; @@ -234,6 +237,9 @@ public static void copyMemory( System.arraycopy((byte[]) src, toIntIndex(srcOffset), (byte[]) dst, toIntIndex(dstOffset), len); return; } + if (copySamePrimitiveArray(src, srcOffset, dst, dstOffset, len)) { + return; + } if (!isPrimitiveArray(src) || !isPrimitiveArray(dst)) { throw unsupportedObjectMemory(); } @@ -248,10 +254,78 @@ public static void copyMemory( } } + private static boolean copySamePrimitiveArray( + Object src, long srcOffset, Object dst, long dstOffset, int len) { + if (src.getClass() != dst.getClass()) { + return false; + } + if (src instanceof boolean[]) { + System.arraycopy((boolean[]) src, toIntIndex(srcOffset), (boolean[]) dst, toIntIndex(dstOffset), len); + return true; + } else if (src instanceof char[] && aligned(srcOffset, dstOffset, len, Character.BYTES)) { + System.arraycopy( + (char[]) src, + toIntIndex(srcOffset / Character.BYTES), + (char[]) dst, + toIntIndex(dstOffset / Character.BYTES), + len / Character.BYTES); + return true; + } else if (src instanceof short[] && aligned(srcOffset, dstOffset, len, Short.BYTES)) { + System.arraycopy( + (short[]) src, + toIntIndex(srcOffset / Short.BYTES), + (short[]) dst, + toIntIndex(dstOffset / Short.BYTES), + len / Short.BYTES); + return true; + } else if (src instanceof int[] && aligned(srcOffset, dstOffset, len, Integer.BYTES)) { + System.arraycopy( + (int[]) src, + toIntIndex(srcOffset / Integer.BYTES), + (int[]) dst, + toIntIndex(dstOffset / Integer.BYTES), + len / Integer.BYTES); + return true; + } else if (src instanceof long[] && aligned(srcOffset, dstOffset, len, Long.BYTES)) { + System.arraycopy( + (long[]) src, + toIntIndex(srcOffset / Long.BYTES), + (long[]) dst, + toIntIndex(dstOffset / Long.BYTES), + len / Long.BYTES); + return true; + } else if (src instanceof float[] && aligned(srcOffset, dstOffset, len, Float.BYTES)) { + System.arraycopy( + (float[]) src, + toIntIndex(srcOffset / Float.BYTES), + (float[]) dst, + toIntIndex(dstOffset / Float.BYTES), + len / Float.BYTES); + return true; + } else if (src instanceof double[] && aligned(srcOffset, dstOffset, len, Double.BYTES)) { + System.arraycopy( + (double[]) src, + toIntIndex(srcOffset / Double.BYTES), + (double[]) dst, + toIntIndex(dstOffset / Double.BYTES), + len / Double.BYTES); + return true; + } + return false; + } + + public static Object[] copyObjectArray(Object[] arr) { + Object[] objects = new Object[arr.length]; + System.arraycopy(arr, 0, objects, 0, arr.length); + return objects; + } + /** Create an instance of type. This method does not call constructor. */ public static T newInstance(Class type) { throw new UnsupportedOperationException( - "Constructor-bypassing allocation is unsupported on JDK25 without sun.misc.Unsafe"); + "Constructor-bypassing allocation is unsupported on JDK25 without sun.misc.Unsafe; " + + "use a constructor-based serializer path for " + + type); } private static int getIntFromArray(Object object, long offset) { @@ -386,6 +460,10 @@ private static boolean isPrimitiveArray(Object object) { return cls.isArray() && cls.getComponentType().isPrimitive(); } + private static boolean aligned(long srcOffset, long dstOffset, int len, int width) { + return srcOffset % width == 0 && dstOffset % width == 0 && len % width == 0; + } + private static int toIntIndex(long offset) { if (offset < 0 || offset > Integer.MAX_VALUE) { throw new IndexOutOfBoundsException("offset out of int range: " + offset); diff --git a/java/fory-core/src/main/java25/org/apache/fory/platform/internal/DefineClass.java b/java/fory-core/src/main/java25/org/apache/fory/platform/internal/DefineClass.java new file mode 100644 index 0000000000..2c4e62358f --- /dev/null +++ b/java/fory-core/src/main/java25/org/apache/fory/platform/internal/DefineClass.java @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.platform.internal; + +import java.lang.invoke.MethodHandle; +import java.lang.invoke.MethodHandles; +import java.lang.invoke.MethodHandles.Lookup; +import java.lang.invoke.MethodType; +import java.security.ProtectionDomain; +import org.apache.fory.annotation.Internal; +import org.apache.fory.platform.AndroidSupport; +import org.apache.fory.platform.JdkVersion; +import org.apache.fory.util.Preconditions; + +/** A class to define bytecode as a class. */ +@Internal +public class DefineClass { + private static volatile MethodHandle classloaderDefineClassHandle; + + public static Class defineClass( + String className, + Class neighbor, + ClassLoader loader, + ProtectionDomain domain, + byte[] bytecodes) { + if (AndroidSupport.IS_ANDROID) { + throw new UnsupportedOperationException( + "Runtime bytecode loading is unsupported on Android."); + } + Preconditions.checkNotNull(loader); + Preconditions.checkArgument(JdkVersion.MAJOR_VERSION >= 8); + if (neighbor != null && JdkVersion.MAJOR_VERSION >= 9) { + // Classes in bytecode must be in same package as lookup class. + MethodHandles.Lookup lookup = MethodHandles.lookup(); + _JDKAccess.addReads(_JDKAccess.getModule(DefineClass.class), _JDKAccess.getModule(neighbor)); + lookup = _Lookup.privateLookupIn(neighbor, lookup); + return _Lookup.defineClass(lookup, bytecodes); + } + if (classloaderDefineClassHandle == null) { + MethodHandles.Lookup lookup = _JDKAccess._trustedLookup(ClassLoader.class); + try { + classloaderDefineClassHandle = + lookup.findVirtual( + ClassLoader.class, + "defineClass", + MethodType.methodType( + Class.class, + String.class, + byte[].class, + int.class, + int.class, + ProtectionDomain.class)); + } catch (NoSuchMethodException | IllegalAccessException e) { + throw new RuntimeException(e); + } + } + try { + return (Class) + classloaderDefineClassHandle.invokeWithArguments( + loader, className, bytecodes, 0, bytecodes.length, domain); + } catch (Throwable e) { + throw new RuntimeException(e); + } + } + + public static Class defineHiddenNestmate(Class neighbor, byte[] bytecodes) { + if (AndroidSupport.IS_ANDROID) { + throw new UnsupportedOperationException( + "Runtime bytecode loading is unsupported on Android."); + } + Preconditions.checkNotNull(neighbor); + Preconditions.checkNotNull(bytecodes); + try { + Lookup lookup = _Lookup.privateLookupIn(neighbor, MethodHandles.lookup()); + return lookup + .defineHiddenClass(bytecodes, true, Lookup.ClassOption.NESTMATE) + .lookupClass(); + } catch (IllegalAccessException | IllegalStateException e) { + Module module = neighbor.getModule(); + Package pkg = neighbor.getPackage(); + String packageName = pkg == null ? "" : pkg.getName(); + throw new IllegalStateException( + "Cannot define hidden nestmate for " + + neighbor.getName() + + " because package " + + packageName + + " in module " + + module.getName() + + " is not open to org.apache.fory.core,org.apache.fory.format", + e); + } + } +} diff --git a/java/fory-core/src/main/java25/org/apache/fory/platform/internal/_JDKAccess.java b/java/fory-core/src/main/java25/org/apache/fory/platform/internal/_JDKAccess.java index 7bdb2ac158..306b782642 100644 --- a/java/fory-core/src/main/java25/org/apache/fory/platform/internal/_JDKAccess.java +++ b/java/fory-core/src/main/java25/org/apache/fory/platform/internal/_JDKAccess.java @@ -21,7 +21,9 @@ import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; +import java.io.ObjectStreamClass; import java.lang.invoke.CallSite; +import java.lang.invoke.LambdaConversionException; import java.lang.invoke.LambdaMetafactory; import java.lang.invoke.MethodHandle; import java.lang.invoke.MethodHandles; @@ -30,9 +32,14 @@ import java.lang.invoke.VarHandle; import java.lang.reflect.Field; import java.lang.reflect.Method; +import java.lang.reflect.Proxy; +import java.nio.charset.StandardCharsets; import java.util.HashMap; import java.util.Map; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; import java.util.function.BiConsumer; +import java.util.function.BiFunction; import java.util.function.Consumer; import java.util.function.Function; import java.util.function.Predicate; @@ -60,6 +67,13 @@ public class _JDKAccess { public static final boolean IS_OPEN_J9; public static final Unsafe UNSAFE = null; public static final boolean JDK_INTERNAL_FIELD_ACCESS; + public static final boolean JDK_LANG_FIELD_ACCESS; + public static final boolean JDK_STRING_FIELD_ACCESS; + public static final boolean JDK_BYTE_ARRAY_STREAM_FIELD_ACCESS; + public static final boolean JDK_OBJECT_STREAM_FIELD_ACCESS; + public static final boolean JDK_COLLECTION_FIELD_ACCESS; + public static final boolean JDK_CONCURRENT_FIELD_ACCESS; + public static final boolean JDK_PROXY_FIELD_ACCESS; public static final Class _INNER_UNSAFE_CLASS = null; public static final Object _INNER_UNSAFE = null; @@ -68,9 +82,10 @@ public class _JDKAccess { public static final boolean STRING_VALUE_FIELD_IS_CHARS; public static final boolean STRING_VALUE_FIELD_IS_BYTES; public static final boolean STRING_HAS_COUNT_OFFSET; - private static final long STRING_VALUE_FIELD_OFFSET = -1; - private static final long STRING_COUNT_FIELD_OFFSET = -1; - private static final long STRING_OFFSET_FIELD_OFFSET = -1; + public static final long STRING_VALUE_FIELD_OFFSET = -1; + public static final long STRING_COUNT_FIELD_OFFSET = -1; + public static final long STRING_OFFSET_FIELD_OFFSET = -1; + public static final long STRING_CODER_FIELD_OFFSET = -1; private static final VarHandle STRING_VALUE_HANDLE; private static final VarHandle STRING_CODER_HANDLE; private static final VarHandle STRING_COUNT_HANDLE; @@ -80,6 +95,11 @@ public class _JDKAccess { private static final VarHandle BIS_BUF_HANDLE; private static final VarHandle BIS_POS_HANDLE; private static final VarHandle BIS_COUNT_HANDLE; + private static final VarHandle OSC_WRITE_OBJECT_METHOD_HANDLE; + private static final VarHandle OSC_READ_OBJECT_METHOD_HANDLE; + private static final VarHandle OSC_READ_OBJECT_NO_DATA_METHOD_HANDLE; + private static final VarHandle OSC_WRITE_REPLACE_METHOD_HANDLE; + private static final VarHandle OSC_READ_RESOLVE_METHOD_HANDLE; static { String jmvName = System.getProperty("java.vm.name", ""); @@ -103,17 +123,43 @@ public class _JDKAccess { } else { STRING_HAS_COUNT_OFFSET = false; } - FieldHandles handles = initFieldHandles(valueField.getType(), countField, offsetField); - JDK_INTERNAL_FIELD_ACCESS = handles != null; - STRING_VALUE_HANDLE = handles == null ? null : handles.stringValue; - STRING_CODER_HANDLE = handles == null ? null : handles.stringCoder; - STRING_COUNT_HANDLE = handles == null ? null : handles.stringCount; - STRING_OFFSET_HANDLE = handles == null ? null : handles.stringOffset; - BAS_BUF_HANDLE = handles == null ? null : handles.basBuf; - BAS_COUNT_HANDLE = handles == null ? null : handles.basCount; - BIS_BUF_HANDLE = handles == null ? null : handles.bisBuf; - BIS_POS_HANDLE = handles == null ? null : handles.bisPos; - BIS_COUNT_HANDLE = handles == null ? null : handles.bisCount; + + StringHandles stringHandles = initStringHandles(valueField.getType(), countField, offsetField); + StreamHandles streamHandles = initStreamHandles(); + ObjectStreamHandles objectStreamHandles = initObjectStreamHandles(); + + JDK_LANG_FIELD_ACCESS = canOpen(String.class); + JDK_STRING_FIELD_ACCESS = stringHandles != null; + JDK_BYTE_ARRAY_STREAM_FIELD_ACCESS = streamHandles != null; + JDK_OBJECT_STREAM_FIELD_ACCESS = objectStreamHandles != null; + JDK_COLLECTION_FIELD_ACCESS = canOpen("java.util.Collections$SynchronizedCollection"); + JDK_CONCURRENT_FIELD_ACCESS = + canOpen(ArrayBlockingQueue.class) && canOpen(LinkedBlockingQueue.class); + JDK_PROXY_FIELD_ACCESS = canOpen(Proxy.class); + JDK_INTERNAL_FIELD_ACCESS = + JDK_STRING_FIELD_ACCESS + && JDK_BYTE_ARRAY_STREAM_FIELD_ACCESS + && JDK_OBJECT_STREAM_FIELD_ACCESS; + + STRING_VALUE_HANDLE = stringHandles == null ? null : stringHandles.value; + STRING_CODER_HANDLE = stringHandles == null ? null : stringHandles.coder; + STRING_COUNT_HANDLE = stringHandles == null ? null : stringHandles.count; + STRING_OFFSET_HANDLE = stringHandles == null ? null : stringHandles.offset; + BAS_BUF_HANDLE = streamHandles == null ? null : streamHandles.basBuf; + BAS_COUNT_HANDLE = streamHandles == null ? null : streamHandles.basCount; + BIS_BUF_HANDLE = streamHandles == null ? null : streamHandles.bisBuf; + BIS_POS_HANDLE = streamHandles == null ? null : streamHandles.bisPos; + BIS_COUNT_HANDLE = streamHandles == null ? null : streamHandles.bisCount; + OSC_WRITE_OBJECT_METHOD_HANDLE = + objectStreamHandles == null ? null : objectStreamHandles.writeObjectMethod; + OSC_READ_OBJECT_METHOD_HANDLE = + objectStreamHandles == null ? null : objectStreamHandles.readObjectMethod; + OSC_READ_OBJECT_NO_DATA_METHOD_HANDLE = + objectStreamHandles == null ? null : objectStreamHandles.readObjectNoDataMethod; + OSC_WRITE_REPLACE_METHOD_HANDLE = + objectStreamHandles == null ? null : objectStreamHandles.writeReplaceMethod; + OSC_READ_RESOLVE_METHOD_HANDLE = + objectStreamHandles == null ? null : objectStreamHandles.readResolveMethod; } catch (NoSuchFieldException e) { throw new RuntimeException(e); } @@ -127,30 +173,31 @@ private static Field getStringFieldNullable(String fieldName) { } } - private static FieldHandles initFieldHandles( + private static StringHandles initStringHandles( Class stringValueType, Field countField, Field offsetField) { try { Lookup stringLookup = MethodHandles.privateLookupIn(String.class, MethodHandles.lookup()); - VarHandle stringValue = stringLookup.findVarHandle(String.class, "value", stringValueType); - VarHandle stringCoder = + return new StringHandles( + stringLookup.findVarHandle(String.class, "value", stringValueType), STRING_VALUE_FIELD_IS_BYTES ? stringLookup.findVarHandle(String.class, "coder", byte.class) - : null; - VarHandle stringCount = - countField == null ? null : stringLookup.findVarHandle(String.class, "count", int.class); - VarHandle stringOffset = + : null, + countField == null ? null : stringLookup.findVarHandle(String.class, "count", int.class), offsetField == null ? null - : stringLookup.findVarHandle(String.class, "offset", int.class); + : stringLookup.findVarHandle(String.class, "offset", int.class)); + } catch (Throwable ignored) { + return null; + } + } + + private static StreamHandles initStreamHandles() { + try { Lookup basLookup = MethodHandles.privateLookupIn(ByteArrayOutputStream.class, MethodHandles.lookup()); Lookup bisLookup = MethodHandles.privateLookupIn(ByteArrayInputStream.class, MethodHandles.lookup()); - return new FieldHandles( - stringValue, - stringCoder, - stringCount, - stringOffset, + return new StreamHandles( basLookup.findVarHandle(ByteArrayOutputStream.class, "buf", byte[].class), basLookup.findVarHandle(ByteArrayOutputStream.class, "count", int.class), bisLookup.findVarHandle(ByteArrayInputStream.class, "buf", byte[].class), @@ -161,31 +208,65 @@ private static FieldHandles initFieldHandles( } } - private static class FieldHandles { - private final VarHandle stringValue; - private final VarHandle stringCoder; - private final VarHandle stringCount; - private final VarHandle stringOffset; + private static ObjectStreamHandles initObjectStreamHandles() { + try { + Lookup oscLookup = + MethodHandles.privateLookupIn(ObjectStreamClass.class, MethodHandles.lookup()); + return new ObjectStreamHandles( + oscLookup.findVarHandle(ObjectStreamClass.class, "writeObjectMethod", Method.class), + oscLookup.findVarHandle(ObjectStreamClass.class, "readObjectMethod", Method.class), + oscLookup.findVarHandle(ObjectStreamClass.class, "readObjectNoDataMethod", Method.class), + oscLookup.findVarHandle(ObjectStreamClass.class, "writeReplaceMethod", Method.class), + oscLookup.findVarHandle(ObjectStreamClass.class, "readResolveMethod", Method.class)); + } catch (Throwable ignored) { + return null; + } + } + + private static boolean canOpen(String className) { + try { + return canOpen(Class.forName(className)); + } catch (Throwable ignored) { + return false; + } + } + + private static boolean canOpen(Class type) { + try { + MethodHandles.privateLookupIn(type, MethodHandles.lookup()); + return true; + } catch (Throwable ignored) { + return false; + } + } + + private static class StringHandles { + private final VarHandle value; + private final VarHandle coder; + private final VarHandle count; + private final VarHandle offset; + + private StringHandles(VarHandle value, VarHandle coder, VarHandle count, VarHandle offset) { + this.value = value; + this.coder = coder; + this.count = count; + this.offset = offset; + } + } + + private static class StreamHandles { private final VarHandle basBuf; private final VarHandle basCount; private final VarHandle bisBuf; private final VarHandle bisPos; private final VarHandle bisCount; - private FieldHandles( - VarHandle stringValue, - VarHandle stringCoder, - VarHandle stringCount, - VarHandle stringOffset, + private StreamHandles( VarHandle basBuf, VarHandle basCount, VarHandle bisBuf, VarHandle bisPos, VarHandle bisCount) { - this.stringValue = stringValue; - this.stringCoder = stringCoder; - this.stringCount = stringCount; - this.stringOffset = stringOffset; this.basBuf = basBuf; this.basCount = basCount; this.bisBuf = bisBuf; @@ -194,13 +275,40 @@ private FieldHandles( } } + private static class ObjectStreamHandles { + private final VarHandle writeObjectMethod; + private final VarHandle readObjectMethod; + private final VarHandle readObjectNoDataMethod; + private final VarHandle writeReplaceMethod; + private final VarHandle readResolveMethod; + + private ObjectStreamHandles( + VarHandle writeObjectMethod, + VarHandle readObjectMethod, + VarHandle readObjectNoDataMethod, + VarHandle writeReplaceMethod, + VarHandle readResolveMethod) { + this.writeObjectMethod = writeObjectMethod; + this.readObjectMethod = readObjectMethod; + this.readObjectNoDataMethod = readObjectNoDataMethod; + this.writeReplaceMethod = writeReplaceMethod; + this.readResolveMethod = readResolveMethod; + } + } + + // The root native-image configuration names these root lazy helpers. Keep same-named JDK25 + // shadows so multi-release class lookup does not fall back to the root Unsafe offset helpers. + private static class StringCoderField {} + + private static class ByteArrayStreamFields {} + public static Object getStringValue(String value) { - checkInternalFieldAccess("String.value"); + checkStringAccess("String.value"); return STRING_VALUE_HANDLE.get(value); } public static byte getStringCoder(String value) { - checkInternalFieldAccess("String.coder"); + checkStringAccess("String.coder"); if (STRING_CODER_HANDLE == null) { throw new UnsupportedOperationException("String.coder is not available on this JDK"); } @@ -208,7 +316,7 @@ public static byte getStringCoder(String value) { } public static int getStringOffset(String value) { - checkInternalFieldAccess("String.offset"); + checkStringAccess("String.offset"); if (STRING_OFFSET_HANDLE == null) { throw new UnsupportedOperationException("String.offset is not available on this JDK"); } @@ -216,7 +324,7 @@ public static int getStringOffset(String value) { } public static int getStringCount(String value) { - checkInternalFieldAccess("String.count"); + checkStringAccess("String.count"); if (STRING_COUNT_HANDLE == null) { throw new UnsupportedOperationException("String.count is not available on this JDK"); } @@ -233,9 +341,181 @@ public static Lookup _trustedLookup(Class objectClass) { return lookupCache.get(objectClass, () -> _Lookup._trustedLookup(objectClass)); } + private static final byte LATIN1 = 0; + private static final Byte LATIN1_BOXED = LATIN1; + private static final byte UTF16 = 1; + private static final Byte UTF16_BOXED = UTF16; + private static final Lookup STRING_LOOK_UP = + JDK_STRING_FIELD_ACCESS ? _trustedLookup(String.class) : null; + private static final MethodHandle STRING_ZERO_COPY_CTR_HANDLE = + JDK_STRING_FIELD_ACCESS ? getJavaStringZeroCopyCtrHandle() : null; + private static final BiFunction CHARS_STRING_ZERO_COPY_CTR = + JDK_STRING_FIELD_ACCESS ? getCharsStringZeroCopyCtr() : null; + private static final BiFunction BYTES_STRING_ZERO_COPY_CTR = + JDK_STRING_FIELD_ACCESS ? getBytesStringZeroCopyCtr() : null; + private static final Function LATIN_BYTES_STRING_ZERO_COPY_CTR = + JDK_STRING_FIELD_ACCESS ? getLatinBytesStringZeroCopyCtr() : null; + + public static String newCharsStringZeroCopy(char[] data) { + if (!JDK_STRING_FIELD_ACCESS) { + return new String(data); + } + if (!STRING_VALUE_FIELD_IS_CHARS) { + throw new IllegalStateException("String value isn't char[], current java isn't supported"); + } + if (CHARS_STRING_ZERO_COPY_CTR == null) { + return newCharsStringByHandle(data); + } + return CHARS_STRING_ZERO_COPY_CTR.apply(data, Boolean.TRUE); + } + + private static String newCharsStringByHandle(char[] data) { + MethodHandle handle = STRING_ZERO_COPY_CTR_HANDLE; + if (handle == null) { + return new String(data); + } + try { + return (String) handle.invokeExact(data, true); + } catch (Throwable ignored) { + return new String(data); + } + } + + public static String newBytesStringZeroCopy(byte coder, byte[] data) { + if (!JDK_STRING_FIELD_ACCESS) { + return newBytesStringSlow(coder, data); + } + if (coder == LATIN1) { + if (LATIN_BYTES_STRING_ZERO_COPY_CTR != null) { + return LATIN_BYTES_STRING_ZERO_COPY_CTR.apply(data); + } else if (BYTES_STRING_ZERO_COPY_CTR == null) { + return newBytesStringByHandle(coder, data); + } + return BYTES_STRING_ZERO_COPY_CTR.apply(data, LATIN1_BOXED); + } else if (coder == UTF16) { + if (BYTES_STRING_ZERO_COPY_CTR == null) { + return newBytesStringByHandle(coder, data); + } + return BYTES_STRING_ZERO_COPY_CTR.apply(data, UTF16_BOXED); + } else { + if (BYTES_STRING_ZERO_COPY_CTR == null) { + return newBytesStringSlow(coder, data); + } + return BYTES_STRING_ZERO_COPY_CTR.apply(data, coder); + } + } + + private static String newBytesStringSlow(byte coder, byte[] data) { + if (coder == LATIN1) { + return new String(data, StandardCharsets.ISO_8859_1); + } else if (coder == UTF16) { + char[] chars = new char[data.length >> 1]; + for (int i = 0, j = 0; i < data.length; i += 2) { + chars[j++] = (char) ((data[i] & 0xff) | ((data[i + 1] & 0xff) << 8)); + } + return new String(chars); + } else { + return new String(data, StandardCharsets.UTF_8); + } + } + + private static String newBytesStringByHandle(byte coder, byte[] data) { + MethodHandle handle = STRING_ZERO_COPY_CTR_HANDLE; + if (handle == null) { + return newBytesStringSlow(coder, data); + } + try { + return (String) handle.invokeExact(data, coder); + } catch (Throwable ignored) { + return newBytesStringSlow(coder, data); + } + } + + private static BiFunction getCharsStringZeroCopyCtr() { + if (!STRING_VALUE_FIELD_IS_CHARS) { + return null; + } + MethodHandle handle = STRING_ZERO_COPY_CTR_HANDLE; + if (handle == null) { + return null; + } + try { + CallSite callSite = + LambdaMetafactory.metafactory( + STRING_LOOK_UP, + "apply", + MethodType.methodType(BiFunction.class), + handle.type().generic(), + handle, + handle.type()); + return (BiFunction) callSite.getTarget().invokeExact(); + } catch (Throwable e) { + return null; + } + } + + private static BiFunction getBytesStringZeroCopyCtr() { + if (!STRING_VALUE_FIELD_IS_BYTES) { + return null; + } + MethodHandle handle = STRING_ZERO_COPY_CTR_HANDLE; + if (handle == null) { + return null; + } + try { + MethodType instantiatedMethodType = + MethodType.methodType(handle.type().returnType(), new Class[] {byte[].class, Byte.class}); + CallSite callSite = + LambdaMetafactory.metafactory( + STRING_LOOK_UP, + "apply", + MethodType.methodType(BiFunction.class), + handle.type().generic(), + handle, + instantiatedMethodType); + return (BiFunction) callSite.getTarget().invokeExact(); + } catch (Throwable e) { + return null; + } + } + + private static Function getLatinBytesStringZeroCopyCtr() { + if (!STRING_VALUE_FIELD_IS_BYTES || STRING_LOOK_UP == null) { + return null; + } + try { + Class clazz = Class.forName("java.lang.StringCoding"); + Lookup caller = STRING_LOOK_UP.in(clazz); + MethodHandle handle = + caller.findStatic( + clazz, "newStringLatin1", MethodType.methodType(String.class, byte[].class)); + return makeFunction(caller, handle, Function.class); + } catch (Throwable e) { + return null; + } + } + + private static MethodHandle getJavaStringZeroCopyCtrHandle() { + Preconditions.checkArgument(JdkVersion.MAJOR_VERSION >= 8); + if (STRING_LOOK_UP == null) { + return null; + } + try { + if (STRING_VALUE_FIELD_IS_CHARS) { + return STRING_LOOK_UP.findConstructor( + String.class, MethodType.methodType(void.class, char[].class, boolean.class)); + } else { + return STRING_LOOK_UP.findConstructor( + String.class, MethodType.methodType(void.class, byte[].class, byte.class)); + } + } catch (Exception e) { + return null; + } + } + public static void wrap(ByteArrayOutputStream stream, MemoryBuffer buffer) { Preconditions.checkNotNull(stream); - checkInternalFieldAccess("ByteArrayOutputStream"); + checkByteArrayStreamAccess("ByteArrayOutputStream"); byte[] buf = (byte[]) BAS_BUF_HANDLE.get(stream); int count = (int) BAS_COUNT_HANDLE.get(stream); buffer.pointTo(buf, 0, buf.length); @@ -244,7 +524,7 @@ public static void wrap(ByteArrayOutputStream stream, MemoryBuffer buffer) { public static void wrap(MemoryBuffer buffer, ByteArrayOutputStream stream) { Preconditions.checkNotNull(stream); - checkInternalFieldAccess("ByteArrayOutputStream"); + checkByteArrayStreamAccess("ByteArrayOutputStream"); byte[] bytes = buffer.getHeapMemory(); Preconditions.checkNotNull(bytes); BAS_BUF_HANDLE.set(stream, bytes); @@ -253,7 +533,7 @@ public static void wrap(MemoryBuffer buffer, ByteArrayOutputStream stream) { public static void wrap(ByteArrayInputStream stream, MemoryBuffer buffer) { Preconditions.checkNotNull(stream); - checkInternalFieldAccess("ByteArrayInputStream"); + checkByteArrayStreamAccess("ByteArrayInputStream"); byte[] buf = (byte[]) BIS_BUF_HANDLE.get(stream); int count = (int) BIS_COUNT_HANDLE.get(stream); int pos = (int) BIS_POS_HANDLE.get(stream); @@ -261,12 +541,57 @@ public static void wrap(ByteArrayInputStream stream, MemoryBuffer buffer) { buffer.readerIndex(pos); } - private static void checkInternalFieldAccess(String target) { - if (!JDK_INTERNAL_FIELD_ACCESS) { + public static Method getObjectStreamClassWriteObjectMethod(ObjectStreamClass objectStreamClass) { + checkObjectStreamAccess("ObjectStreamClass.writeObjectMethod"); + return (Method) OSC_WRITE_OBJECT_METHOD_HANDLE.get(objectStreamClass); + } + + public static Method getObjectStreamClassReadObjectMethod(ObjectStreamClass objectStreamClass) { + checkObjectStreamAccess("ObjectStreamClass.readObjectMethod"); + return (Method) OSC_READ_OBJECT_METHOD_HANDLE.get(objectStreamClass); + } + + public static Method getObjectStreamClassReadObjectNoDataMethod( + ObjectStreamClass objectStreamClass) { + checkObjectStreamAccess("ObjectStreamClass.readObjectNoDataMethod"); + return (Method) OSC_READ_OBJECT_NO_DATA_METHOD_HANDLE.get(objectStreamClass); + } + + public static Method getObjectStreamClassWriteReplaceMethod( + ObjectStreamClass objectStreamClass) { + checkObjectStreamAccess("ObjectStreamClass.writeReplaceMethod"); + return (Method) OSC_WRITE_REPLACE_METHOD_HANDLE.get(objectStreamClass); + } + + public static Method getObjectStreamClassReadResolveMethod(ObjectStreamClass objectStreamClass) { + checkObjectStreamAccess("ObjectStreamClass.readResolveMethod"); + return (Method) OSC_READ_RESOLVE_METHOD_HANDLE.get(objectStreamClass); + } + + private static void checkStringAccess(String target) { + if (!JDK_STRING_FIELD_ACCESS) { + throw new UnsupportedOperationException( + target + + " private access is unavailable; open java.base/java.lang to " + + "org.apache.fory.core,org.apache.fory.format"); + } + } + + private static void checkByteArrayStreamAccess(String target) { + if (!JDK_BYTE_ARRAY_STREAM_FIELD_ACCESS) { + throw new UnsupportedOperationException( + target + + " private access is unavailable; open java.base/java.io to " + + "org.apache.fory.core,org.apache.fory.format"); + } + } + + private static void checkObjectStreamAccess(String target) { + if (!JDK_OBJECT_STREAM_FIELD_ACCESS) { throw new UnsupportedOperationException( target - + " private access is unavailable; open java.base/java.lang and java.base/java.io " - + "to org.apache.fory.core"); + + " private access is unavailable; open java.base/java.io to " + + "org.apache.fory.core,org.apache.fory.format"); } } @@ -451,6 +776,8 @@ public static Object makeGetterFunction( handle, handle.type()); return callSite.getTarget().invoke(); + } catch (LambdaConversionException e) { + return makeGetterFallback(handle, returnType); } catch (ClassNotFoundException | NoClassDefFoundError e) { return makeGetterFunction(lookup, handle, Object.class); } catch (Throwable e) { @@ -458,6 +785,90 @@ public static Object makeGetterFunction( } } + private static Object makeGetterFallback(MethodHandle handle, Class returnType) { + if (returnType == boolean.class) { + return (Predicate) + value -> { + try { + return (boolean) handle.invoke(value); + } catch (Throwable e) { + throw ExceptionUtils.throwException(e); + } + }; + } else if (returnType == byte.class) { + return (ToByteFunction) + value -> { + try { + return (byte) handle.invoke(value); + } catch (Throwable e) { + throw ExceptionUtils.throwException(e); + } + }; + } else if (returnType == char.class) { + return (ToCharFunction) + value -> { + try { + return (char) handle.invoke(value); + } catch (Throwable e) { + throw ExceptionUtils.throwException(e); + } + }; + } else if (returnType == short.class) { + return (ToShortFunction) + value -> { + try { + return (short) handle.invoke(value); + } catch (Throwable e) { + throw ExceptionUtils.throwException(e); + } + }; + } else if (returnType == int.class) { + return (ToIntFunction) + value -> { + try { + return (int) handle.invoke(value); + } catch (Throwable e) { + throw ExceptionUtils.throwException(e); + } + }; + } else if (returnType == long.class) { + return (ToLongFunction) + value -> { + try { + return (long) handle.invoke(value); + } catch (Throwable e) { + throw ExceptionUtils.throwException(e); + } + }; + } else if (returnType == float.class) { + return (ToFloatFunction) + value -> { + try { + return (float) handle.invoke(value); + } catch (Throwable e) { + throw ExceptionUtils.throwException(e); + } + }; + } else if (returnType == double.class) { + return (ToDoubleFunction) + value -> { + try { + return (double) handle.invoke(value); + } catch (Throwable e) { + throw ExceptionUtils.throwException(e); + } + }; + } + return (Function) + value -> { + try { + return handle.invoke(value); + } catch (Throwable e) { + throw ExceptionUtils.throwException(e); + } + }; + } + public static Object getModule(Class cls) { Preconditions.checkArgument(JdkVersion.MAJOR_VERSION >= 9); return cls.getModule(); @@ -467,4 +878,8 @@ public static Object addReads(Object thisModule, Object otherModule) { Preconditions.checkArgument(JdkVersion.MAJOR_VERSION >= 9); return ((Module) thisModule).addReads((Module) otherModule); } + + public static Lookup privateLookupIn(Class targetClass, Lookup caller) { + return _Lookup.privateLookupIn(targetClass, caller); + } } diff --git a/java/fory-core/src/main/java25/org/apache/fory/platform/internal/_Lookup.java b/java/fory-core/src/main/java25/org/apache/fory/platform/internal/_Lookup.java index 0d3a669321..b7a0b8272f 100644 --- a/java/fory-core/src/main/java25/org/apache/fory/platform/internal/_Lookup.java +++ b/java/fory-core/src/main/java25/org/apache/fory/platform/internal/_Lookup.java @@ -35,6 +35,11 @@ public static Lookup _trustedLookup(Class objectClass) { public static Lookup privateLookupIn(Class targetClass, Lookup caller) { try { + Module foryModule = _Lookup.class.getModule(); + Module targetModule = targetClass.getModule(); + if (foryModule != targetModule) { + foryModule.addReads(targetModule); + } return MethodHandles.privateLookupIn(targetClass, caller); } catch (IllegalAccessException e) { throw new IllegalStateException(privateAccessMessage(targetClass), e); @@ -64,6 +69,6 @@ private static String privateAccessMessage(Class targetClass) { + packageName + " in module " + module.getName() - + " to be open to org.apache.fory.core"; + + " to be open to org.apache.fory.core,org.apache.fory.format"; } } diff --git a/java/fory-core/src/main/java25/org/apache/fory/reflect/FieldAccessor.java b/java/fory-core/src/main/java25/org/apache/fory/reflect/FieldAccessor.java index 3f0e9a2ff7..82c8cc21e9 100644 --- a/java/fory-core/src/main/java25/org/apache/fory/reflect/FieldAccessor.java +++ b/java/fory-core/src/main/java25/org/apache/fory/reflect/FieldAccessor.java @@ -35,6 +35,7 @@ import org.apache.fory.exception.ForyException; import org.apache.fory.platform.AndroidSupport; import org.apache.fory.platform.GraalvmSupport; +import org.apache.fory.platform.internal._JDKAccess; import org.apache.fory.type.TypeUtils; import org.apache.fory.util.Preconditions; import org.apache.fory.util.function.ToByteFunction; @@ -46,8 +47,21 @@ /** Field accessor for primitive types and object types. */ @SuppressWarnings({"unchecked", "rawtypes"}) public abstract class FieldAccessor { + private static final int BOOLEAN_ACCESS = 1; + private static final int BYTE_ACCESS = 2; + private static final int CHAR_ACCESS = 3; + private static final int SHORT_ACCESS = 4; + private static final int INT_ACCESS = 5; + private static final int LONG_ACCESS = 6; + private static final int FLOAT_ACCESS = 7; + private static final int DOUBLE_ACCESS = 8; + private static final int OBJECT_ACCESS = 9; + protected final Field field; + // Kept to preserve the root FieldAccessor shape for already compiled internal subclasses. + // JDK25 access is field-owned through VarHandle/MethodHandle and intentionally never uses it. protected final long fieldOffset; + private final int accessKind; public FieldAccessor(Field field) { this(field, -1); @@ -57,6 +71,29 @@ protected FieldAccessor(Field field, long fieldOffset) { this.field = field; this.fieldOffset = fieldOffset; Preconditions.checkNotNull(field); + this.accessKind = accessKind(field); + } + + private static int accessKind(Field field) { + Class fieldType = field.getType(); + if (fieldType == boolean.class) { + return BOOLEAN_ACCESS; + } else if (fieldType == byte.class) { + return BYTE_ACCESS; + } else if (fieldType == char.class) { + return CHAR_ACCESS; + } else if (fieldType == short.class) { + return SHORT_ACCESS; + } else if (fieldType == int.class) { + return INT_ACCESS; + } else if (fieldType == long.class) { + return LONG_ACCESS; + } else if (fieldType == float.class) { + return FLOAT_ACCESS; + } else if (fieldType == double.class) { + return DOUBLE_ACCESS; + } + return OBJECT_ACCESS; } public abstract Object get(Object obj); @@ -65,6 +102,41 @@ public void set(Object obj, Object value) { throw new UnsupportedOperationException("Unsupported for field " + field); } + public final void copy(Object sourceObject, Object targetObject) { + switch (accessKind) { + case BOOLEAN_ACCESS: + putBoolean(targetObject, getBoolean(sourceObject)); + return; + case BYTE_ACCESS: + putByte(targetObject, getByte(sourceObject)); + return; + case CHAR_ACCESS: + putChar(targetObject, getChar(sourceObject)); + return; + case SHORT_ACCESS: + putShort(targetObject, getShort(sourceObject)); + return; + case INT_ACCESS: + putInt(targetObject, getInt(sourceObject)); + return; + case LONG_ACCESS: + putLong(targetObject, getLong(sourceObject)); + return; + case FLOAT_ACCESS: + putFloat(targetObject, getFloat(sourceObject)); + return; + case DOUBLE_ACCESS: + putDouble(targetObject, getDouble(sourceObject)); + return; + default: + putObject(targetObject, getObject(sourceObject)); + } + } + + public final void copyObject(Object sourceObject, Object targetObject) { + putObject(targetObject, getObject(sourceObject)); + } + public Field getField() { return field; } @@ -133,11 +205,11 @@ public void putDouble(Object targetObject, double value) { set(targetObject, value); } - public void putObject(Object targetObject, Object object) { + public final void putObject(Object targetObject, Object object) { set(targetObject, object); } - public Object getObject(Object targetObject) { + public final Object getObject(Object targetObject) { return get(targetObject); } @@ -179,9 +251,13 @@ public static FieldAccessor createAccessor(Field field) { // generated accessors, or primitive-specific reflection subclasses. return new ReflectionFieldAccessor(field); } - if (GraalvmSupport.isGraalBuildTime()) { + if (GraalvmSupport.isGraalBuildTime() || GraalvmSupport.IN_GRAALVM_NATIVE_IMAGE) { return new GeneratedAccessor(field); } + FieldAccessor hiddenAccessor = HiddenFieldAccessorFactory.create(field); + if (hiddenAccessor != null) { + return hiddenAccessor; + } return createVarHandleAccessor(field); } @@ -240,13 +316,16 @@ private static FieldAccessor createRecordAccessor(Field field) { } private static VarHandle fieldHandle(Field field) { - MethodHandles.Lookup lookup = privateLookup(field); try { - if (Modifier.isStatic(field.getModifiers())) { - return lookup.findStaticVarHandle( - field.getDeclaringClass(), field.getName(), field.getType()); + if (canUsePublicField(field)) { + try { + return findFieldHandle(MethodHandles.publicLookup(), field); + } catch (IllegalAccessException ignored) { + // The package may be opened but not exported. Fall through to privateLookupIn so + // --add-opens still enables access for named-module users. + } } - return lookup.findVarHandle(field.getDeclaringClass(), field.getName(), field.getType()); + return findFieldHandle(privateLookup(field), field); } catch (IllegalAccessException e) { throw accessFailure(field, e); } catch (NoSuchFieldException e) { @@ -255,10 +334,15 @@ private static VarHandle fieldHandle(Field field) { } private static MethodHandle recordGetter(Field field) { - MethodHandles.Lookup lookup = privateLookup(field); try { - return lookup.findVirtual( - field.getDeclaringClass(), field.getName(), MethodType.methodType(field.getType())); + if (Modifier.isPublic(field.getDeclaringClass().getModifiers())) { + try { + return findRecordGetter(MethodHandles.publicLookup(), field); + } catch (IllegalAccessException ignored) { + // The package may be opened but not exported. Fall through to privateLookupIn. + } + } + return findRecordGetter(privateLookup(field), field); } catch (IllegalAccessException e) { throw accessFailure(field, e); } catch (NoSuchMethodException e) { @@ -266,13 +350,29 @@ private static MethodHandle recordGetter(Field field) { } } + private static VarHandle findFieldHandle(MethodHandles.Lookup lookup, Field field) + throws IllegalAccessException, NoSuchFieldException { + if (Modifier.isStatic(field.getModifiers())) { + return lookup.findStaticVarHandle( + field.getDeclaringClass(), field.getName(), field.getType()); + } + return lookup.findVarHandle(field.getDeclaringClass(), field.getName(), field.getType()); + } + + private static MethodHandle findRecordGetter(MethodHandles.Lookup lookup, Field field) + throws IllegalAccessException, NoSuchMethodException { + return lookup.findVirtual( + field.getDeclaringClass(), field.getName(), MethodType.methodType(field.getType())); + } + private static MethodHandles.Lookup privateLookup(Field field) { Class declaringClass = field.getDeclaringClass(); - try { - return MethodHandles.privateLookupIn(declaringClass, MethodHandles.lookup()); - } catch (IllegalAccessException e) { - throw accessFailure(field, e); - } + return _JDKAccess.privateLookupIn(declaringClass, MethodHandles.lookup()); + } + + private static boolean canUsePublicField(Field field) { + return Modifier.isPublic(field.getModifiers()) + && Modifier.isPublic(field.getDeclaringClass().getModifiers()); } private static IllegalStateException accessFailure(Field field, Throwable cause) { @@ -280,6 +380,8 @@ private static IllegalStateException accessFailure(Field field, Throwable cause) Module targetModule = declaringClass.getModule(); Package targetPackage = declaringClass.getPackage(); String packageName = targetPackage == null ? "" : targetPackage.getName(); + String openTarget = + moduleName(targetModule) + (packageName.isEmpty() ? "" : "/" + packageName); return new IllegalStateException( "Cannot access field " + field @@ -288,7 +390,10 @@ private static IllegalStateException accessFailure(Field field, Throwable cause) + " in module " + moduleName(targetModule) + " is not open to " - + moduleName(FieldAccessor.class.getModule()), + + moduleName(FieldAccessor.class.getModule()) + + ". For named modules, open the package with --add-opens=" + + openTarget + + "=org.apache.fory.core,org.apache.fory.format", cause); } @@ -351,6 +456,98 @@ private abstract static class VarHandleAccessor extends FieldAccessor { handle = fieldHandle(field); isStatic = Modifier.isStatic(field.getModifiers()); } + + protected void setReflectively(Object obj, Object value, Throwable cause) { + try { + prepareReflectiveWrite(cause); + field.set(target(obj), value); + } catch (IllegalAccessException | RuntimeException e) { + throw unsupportedWrite(field, e); + } + } + + private Object target(Object obj) { + return isStatic ? null : obj; + } + + private void prepareReflectiveWrite(Throwable cause) { + if (field.getDeclaringClass().getName().startsWith("java.")) { + throw unsupportedWrite(field, cause); + } + field.setAccessible(true); + } + + protected void setBooleanReflectively(Object obj, boolean value, Throwable cause) { + try { + prepareReflectiveWrite(cause); + field.setBoolean(target(obj), value); + } catch (IllegalAccessException | RuntimeException e) { + throw unsupportedWrite(field, e); + } + } + + protected void setByteReflectively(Object obj, byte value, Throwable cause) { + try { + prepareReflectiveWrite(cause); + field.setByte(target(obj), value); + } catch (IllegalAccessException | RuntimeException e) { + throw unsupportedWrite(field, e); + } + } + + protected void setCharReflectively(Object obj, char value, Throwable cause) { + try { + prepareReflectiveWrite(cause); + field.setChar(target(obj), value); + } catch (IllegalAccessException | RuntimeException e) { + throw unsupportedWrite(field, e); + } + } + + protected void setShortReflectively(Object obj, short value, Throwable cause) { + try { + prepareReflectiveWrite(cause); + field.setShort(target(obj), value); + } catch (IllegalAccessException | RuntimeException e) { + throw unsupportedWrite(field, e); + } + } + + protected void setIntReflectively(Object obj, int value, Throwable cause) { + try { + prepareReflectiveWrite(cause); + field.setInt(target(obj), value); + } catch (IllegalAccessException | RuntimeException e) { + throw unsupportedWrite(field, e); + } + } + + protected void setLongReflectively(Object obj, long value, Throwable cause) { + try { + prepareReflectiveWrite(cause); + field.setLong(target(obj), value); + } catch (IllegalAccessException | RuntimeException e) { + throw unsupportedWrite(field, e); + } + } + + protected void setFloatReflectively(Object obj, float value, Throwable cause) { + try { + prepareReflectiveWrite(cause); + field.setFloat(target(obj), value); + } catch (IllegalAccessException | RuntimeException e) { + throw unsupportedWrite(field, e); + } + } + + protected void setDoubleReflectively(Object obj, double value, Throwable cause) { + try { + prepareReflectiveWrite(cause); + field.setDouble(target(obj), value); + } catch (IllegalAccessException | RuntimeException e) { + throw unsupportedWrite(field, e); + } + } } /** Primitive boolean accessor. */ @@ -389,7 +586,7 @@ public void putBoolean(Object obj, boolean value) { handle.set(obj, value); } } catch (UnsupportedOperationException e) { - throw unsupportedWrite(field, e); + setBooleanReflectively(obj, value, e); } } } @@ -467,7 +664,7 @@ public void putByte(Object obj, byte value) { handle.set(obj, value); } } catch (UnsupportedOperationException e) { - throw unsupportedWrite(field, e); + setByteReflectively(obj, value, e); } } } @@ -546,7 +743,7 @@ public void putChar(Object obj, char value) { handle.set(obj, value); } } catch (UnsupportedOperationException e) { - throw unsupportedWrite(field, e); + setCharReflectively(obj, value, e); } } } @@ -624,7 +821,7 @@ public void putShort(Object obj, short value) { handle.set(obj, value); } } catch (UnsupportedOperationException e) { - throw unsupportedWrite(field, e); + setShortReflectively(obj, value, e); } } } @@ -702,7 +899,7 @@ public void putInt(Object obj, int value) { handle.set(obj, value); } } catch (UnsupportedOperationException e) { - throw unsupportedWrite(field, e); + setIntReflectively(obj, value, e); } } } @@ -780,7 +977,7 @@ public void putLong(Object obj, long value) { handle.set(obj, value); } } catch (UnsupportedOperationException e) { - throw unsupportedWrite(field, e); + setLongReflectively(obj, value, e); } } } @@ -858,7 +1055,7 @@ public void putFloat(Object obj, float value) { handle.set(obj, value); } } catch (UnsupportedOperationException e) { - throw unsupportedWrite(field, e); + setFloatReflectively(obj, value, e); } } } @@ -936,7 +1133,7 @@ public void putDouble(Object obj, double value) { handle.set(obj, value); } } catch (UnsupportedOperationException e) { - throw unsupportedWrite(field, e); + setDoubleReflectively(obj, value, e); } } } @@ -1004,7 +1201,7 @@ public void set(Object obj, Object value) { handle.set(obj, value); } } catch (UnsupportedOperationException e) { - throw unsupportedWrite(field, e); + setReflectively(obj, value, e); } } } @@ -1100,7 +1297,7 @@ public void set(Object obj, Object value) { handle.set(obj, value); } } catch (UnsupportedOperationException e) { - throw unsupportedWrite(field, e); + setReflectively(obj, value, e); } catch (RuntimeException e) { throw accessorFailure(field, e); } @@ -1129,7 +1326,7 @@ public void putBoolean(Object obj, boolean value) { handle.set(obj, value); } } catch (UnsupportedOperationException e) { - throw unsupportedWrite(field, e); + setBooleanReflectively(obj, value, e); } catch (RuntimeException e) { throw accessorFailure(field, e); } @@ -1158,7 +1355,7 @@ public void putByte(Object obj, byte value) { handle.set(obj, value); } } catch (UnsupportedOperationException e) { - throw unsupportedWrite(field, e); + setByteReflectively(obj, value, e); } catch (RuntimeException e) { throw accessorFailure(field, e); } @@ -1187,7 +1384,7 @@ public void putChar(Object obj, char value) { handle.set(obj, value); } } catch (UnsupportedOperationException e) { - throw unsupportedWrite(field, e); + setCharReflectively(obj, value, e); } catch (RuntimeException e) { throw accessorFailure(field, e); } @@ -1216,7 +1413,7 @@ public void putShort(Object obj, short value) { handle.set(obj, value); } } catch (UnsupportedOperationException e) { - throw unsupportedWrite(field, e); + setShortReflectively(obj, value, e); } catch (RuntimeException e) { throw accessorFailure(field, e); } @@ -1245,7 +1442,7 @@ public void putInt(Object obj, int value) { handle.set(obj, value); } } catch (UnsupportedOperationException e) { - throw unsupportedWrite(field, e); + setIntReflectively(obj, value, e); } catch (RuntimeException e) { throw accessorFailure(field, e); } @@ -1274,7 +1471,7 @@ public void putLong(Object obj, long value) { handle.set(obj, value); } } catch (UnsupportedOperationException e) { - throw unsupportedWrite(field, e); + setLongReflectively(obj, value, e); } catch (RuntimeException e) { throw accessorFailure(field, e); } @@ -1303,7 +1500,7 @@ public void putFloat(Object obj, float value) { handle.set(obj, value); } } catch (UnsupportedOperationException e) { - throw unsupportedWrite(field, e); + setFloatReflectively(obj, value, e); } catch (RuntimeException e) { throw accessorFailure(field, e); } @@ -1332,7 +1529,7 @@ public void putDouble(Object obj, double value) { handle.set(obj, value); } } catch (UnsupportedOperationException e) { - throw unsupportedWrite(field, e); + setDoubleReflectively(obj, value, e); } catch (RuntimeException e) { throw accessorFailure(field, e); } diff --git a/java/fory-core/src/main/java25/org/apache/fory/reflect/HiddenFieldAccessorFactory.java b/java/fory-core/src/main/java25/org/apache/fory/reflect/HiddenFieldAccessorFactory.java new file mode 100644 index 0000000000..b9d88bbb5d --- /dev/null +++ b/java/fory-core/src/main/java25/org/apache/fory/reflect/HiddenFieldAccessorFactory.java @@ -0,0 +1,559 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.reflect; + +import java.io.ByteArrayOutputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.lang.reflect.Constructor; +import java.lang.reflect.Field; +import java.lang.reflect.Modifier; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; +import org.apache.fory.platform.internal.DefineClass; + +final class HiddenFieldAccessorFactory { + private static final int JAVA5_CLASS_VERSION = 49; + private static final int ACC_PUBLIC = 0x0001; + private static final int ACC_FINAL = 0x0010; + private static final int ACC_SUPER = 0x0020; + private static final String FIELD_ACCESSOR = "org/apache/fory/reflect/FieldAccessor"; + private static final String FIELD_CTR_DESC = "(Ljava/lang/reflect/Field;)V"; + private static final AtomicInteger IDS = new AtomicInteger(); + + private HiddenFieldAccessorFactory() {} + + static FieldAccessor create(Field field) { + if (Modifier.isStatic(field.getModifiers()) || Modifier.isFinal(field.getModifiers())) { + return null; + } + try { + byte[] bytes = new AccessorClass(field).bytes(); + Class accessorClass = DefineClass.defineHiddenNestmate(field.getDeclaringClass(), bytes); + Constructor constructor = accessorClass.getDeclaredConstructor(Field.class); + constructor.setAccessible(true); + return (FieldAccessor) constructor.newInstance(field); + } catch (ReflectiveOperationException | RuntimeException | LinkageError e) { + return null; + } + } + + private static final class AccessorClass { + private final Field field; + private final Class fieldType; + private final String owner; + private final String thisClass; + private final String fieldDesc; + private final Pool pool = new Pool(); + + private int codeUtf8; + private int initName; + private int initDesc; + private int getName; + private int getDesc; + private int setName; + private int setDesc; + private int fieldRef; + private int ownerClass; + private int thisClassIndex; + private int fieldAccessorClass; + private int superCtr; + + private AccessorClass(Field field) { + this.field = field; + this.fieldType = field.getType(); + owner = internalName(field.getDeclaringClass()); + thisClass = owner + "$ForyFieldAccessor$" + IDS.incrementAndGet(); + fieldDesc = descriptor(fieldType); + } + + private byte[] bytes() { + try { + preparePool(); + List methods = methods(); + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + DataOutputStream out = new DataOutputStream(bytes); + out.writeInt(0xCAFEBABE); + out.writeShort(0); + out.writeShort(JAVA5_CLASS_VERSION); + pool.writeTo(out); + out.writeShort(ACC_FINAL | ACC_SUPER); + out.writeShort(thisClassIndex); + out.writeShort(fieldAccessorClass); + out.writeShort(0); + out.writeShort(0); + out.writeShort(methods.size()); + for (MethodDef method : methods) { + method.writeTo(out, codeUtf8); + } + out.writeShort(0); + return bytes.toByteArray(); + } catch (IOException e) { + throw new IllegalStateException("Failed to build field accessor bytecode", e); + } + } + + private void preparePool() { + codeUtf8 = pool.utf8("Code"); + initName = pool.utf8(""); + initDesc = pool.utf8(FIELD_CTR_DESC); + getName = pool.utf8("get"); + getDesc = pool.utf8("(Ljava/lang/Object;)Ljava/lang/Object;"); + setName = pool.utf8("set"); + setDesc = pool.utf8("(Ljava/lang/Object;Ljava/lang/Object;)V"); + thisClassIndex = pool.classInfo(thisClass); + ownerClass = pool.classInfo(owner); + fieldAccessorClass = pool.classInfo(FIELD_ACCESSOR); + superCtr = pool.methodRef(FIELD_ACCESSOR, "", FIELD_CTR_DESC); + fieldRef = pool.fieldRef(owner, field.getName(), fieldDesc); + if (fieldType.isPrimitive()) { + Primitive primitive = Primitive.of(fieldType); + pool.classInfo(primitive.wrapper); + pool.methodRef(primitive.wrapper, "valueOf", primitive.valueOfDesc); + pool.methodRef(primitive.wrapper, primitive.unboxName, primitive.unboxDesc); + } else { + pool.classInfo(castName(fieldType)); + } + } + + private List methods() throws IOException { + List methods = new ArrayList<>(); + methods.add(method(ACC_PUBLIC, initName, initDesc, 2, 2, constructorCode())); + methods.add(method(ACC_PUBLIC, getName, getDesc, 4, 2, objectGetterCode())); + if (fieldType.isPrimitive()) { + Primitive primitive = Primitive.of(fieldType); + methods.add( + method( + ACC_PUBLIC, + pool.utf8(primitive.getterName), + pool.utf8("(Ljava/lang/Object;)" + primitive.desc), + 3, + 2, + primitiveGetterCode(primitive))); + if (!Modifier.isFinal(field.getModifiers())) { + methods.add(method(ACC_PUBLIC, setName, setDesc, 4, 3, objectSetterCode(primitive))); + methods.add( + method( + ACC_PUBLIC, + pool.utf8(primitive.setterName), + pool.utf8("(Ljava/lang/Object;" + primitive.desc + ")V"), + 4, + primitive.maxLocals, + primitiveSetterCode(primitive))); + } + } else { + if (!Modifier.isFinal(field.getModifiers())) { + methods.add(method(ACC_PUBLIC, setName, setDesc, 3, 3, referenceSetterCode())); + } + } + return methods; + } + + private byte[] constructorCode() throws IOException { + Code code = new Code(); + code.u1(0x2A); // aload_0 + code.u1(0x2B); // aload_1 + code.u1(0xB7).u2(superCtr); // invokespecial + code.u1(0xB1); // return + return code.bytes(); + } + + private byte[] objectGetterCode() throws IOException { + Code code = directReadCode(); + if (fieldType.isPrimitive()) { + Primitive primitive = Primitive.of(fieldType); + code.u1(0xB8).u2(pool.methodRef(primitive.wrapper, "valueOf", primitive.valueOfDesc)); + } + code.u1(0xB0); // areturn + return code.bytes(); + } + + private byte[] primitiveGetterCode(Primitive primitive) throws IOException { + Code code = directReadCode(); + code.u1(primitive.returnOpcode); + return code.bytes(); + } + + private Code directReadCode() throws IOException { + Code code = new Code(); + code.u1(0x2B); // aload_1 + code.u1(0xC0).u2(ownerClass); // checkcast + code.u1(0xB4).u2(fieldRef); // getfield + return code; + } + + private byte[] objectSetterCode(Primitive primitive) throws IOException { + Code code = setterPrefix(); + code.u1(0x2C); // aload_2 + code.u1(0xC0).u2(pool.classInfo(primitive.wrapper)); + code.u1(0xB6).u2(pool.methodRef(primitive.wrapper, primitive.unboxName, primitive.unboxDesc)); + code.u1(0xB5).u2(fieldRef); // putfield + code.u1(0xB1); // return + return code.bytes(); + } + + private byte[] primitiveSetterCode(Primitive primitive) throws IOException { + Code code = setterPrefix(); + code.u1(primitive.loadOpcode); + code.u1(0xB5).u2(fieldRef); // putfield + code.u1(0xB1); // return + return code.bytes(); + } + + private byte[] referenceSetterCode() throws IOException { + Code code = setterPrefix(); + code.u1(0x2C); // aload_2 + code.u1(0xC0).u2(pool.classInfo(castName(fieldType))); + code.u1(0xB5).u2(fieldRef); // putfield + code.u1(0xB1); // return + return code.bytes(); + } + + private Code setterPrefix() throws IOException { + Code code = new Code(); + code.u1(0x2B); // aload_1 + code.u1(0xC0).u2(ownerClass); // checkcast + return code; + } + + private MethodDef method( + int access, + int name, + int desc, + int maxStack, + int maxLocals, + byte[] code) + throws IOException { + return new MethodDef(access, name, desc, maxStack, maxLocals, code); + } + } + + private static final class MethodDef { + private final int access; + private final int name; + private final int desc; + private final int maxStack; + private final int maxLocals; + private final byte[] code; + + private MethodDef(int access, int name, int desc, int maxStack, int maxLocals, byte[] code) { + this.access = access; + this.name = name; + this.desc = desc; + this.maxStack = maxStack; + this.maxLocals = maxLocals; + this.code = code; + } + + private void writeTo(DataOutputStream out, int codeUtf8) throws IOException { + out.writeShort(access); + out.writeShort(name); + out.writeShort(desc); + out.writeShort(1); + out.writeShort(codeUtf8); + out.writeInt(12 + code.length); + out.writeShort(maxStack); + out.writeShort(maxLocals); + out.writeInt(code.length); + out.write(code); + out.writeShort(0); + out.writeShort(0); + } + } + + private static String descriptor(Class type) { + if (type == void.class) { + return "V"; + } else if (type == boolean.class) { + return "Z"; + } else if (type == byte.class) { + return "B"; + } else if (type == char.class) { + return "C"; + } else if (type == short.class) { + return "S"; + } else if (type == int.class) { + return "I"; + } else if (type == long.class) { + return "J"; + } else if (type == float.class) { + return "F"; + } else if (type == double.class) { + return "D"; + } else if (type.isArray()) { + return type.getName().replace('.', '/'); + } + return "L" + internalName(type) + ";"; + } + + private static String castName(Class type) { + if (type.isArray()) { + return descriptor(type); + } + return internalName(type); + } + + private static String internalName(Class type) { + return type.getName().replace('.', '/'); + } + + private static final class Code { + private final ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + private final DataOutputStream out = new DataOutputStream(bytes); + + private Code u1(int value) throws IOException { + out.writeByte(value); + return this; + } + + private Code u2(int value) throws IOException { + out.writeShort(value); + return this; + } + + private byte[] bytes() throws IOException { + out.flush(); + return bytes.toByteArray(); + } + } + + private static final class Pool { + private final ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + private final DataOutputStream out = new DataOutputStream(bytes); + private int count = 1; + + private int utf8(String value) { + try { + int index = count++; + out.writeByte(1); + out.writeUTF(value); + return index; + } catch (IOException e) { + throw new IllegalStateException(e); + } + } + + private int classInfo(String name) { + try { + int nameIndex = utf8(name); + int index = count++; + out.writeByte(7); + out.writeShort(nameIndex); + return index; + } catch (IOException e) { + throw new IllegalStateException(e); + } + } + + private int nameAndType(String name, String desc) { + try { + int nameIndex = utf8(name); + int descIndex = utf8(desc); + int index = count++; + out.writeByte(12); + out.writeShort(nameIndex); + out.writeShort(descIndex); + return index; + } catch (IOException e) { + throw new IllegalStateException(e); + } + } + + private int fieldRef(String owner, String name, String desc) { + try { + int ownerIndex = classInfo(owner); + int nameAndTypeIndex = nameAndType(name, desc); + int index = count++; + out.writeByte(9); + out.writeShort(ownerIndex); + out.writeShort(nameAndTypeIndex); + return index; + } catch (IOException e) { + throw new IllegalStateException(e); + } + } + + private int methodRef(String owner, String name, String desc) { + try { + int ownerIndex = classInfo(owner); + int nameAndTypeIndex = nameAndType(name, desc); + int index = count++; + out.writeByte(10); + out.writeShort(ownerIndex); + out.writeShort(nameAndTypeIndex); + return index; + } catch (IOException e) { + throw new IllegalStateException(e); + } + } + + private void writeTo(DataOutputStream target) throws IOException { + out.flush(); + target.writeShort(count); + bytes.writeTo(target); + } + } + + private enum Primitive { + BOOLEAN( + boolean.class, + "Z", + "java/lang/Boolean", + "(Z)Ljava/lang/Boolean;", + "booleanValue", + "()Z", + "getBoolean", + "putBoolean", + 0x1C, + 0xAC, + 3), + BYTE( + byte.class, + "B", + "java/lang/Byte", + "(B)Ljava/lang/Byte;", + "byteValue", + "()B", + "getByte", + "putByte", + 0x1C, + 0xAC, + 3), + CHAR( + char.class, + "C", + "java/lang/Character", + "(C)Ljava/lang/Character;", + "charValue", + "()C", + "getChar", + "putChar", + 0x1C, + 0xAC, + 3), + SHORT( + short.class, + "S", + "java/lang/Short", + "(S)Ljava/lang/Short;", + "shortValue", + "()S", + "getShort", + "putShort", + 0x1C, + 0xAC, + 3), + INT( + int.class, + "I", + "java/lang/Integer", + "(I)Ljava/lang/Integer;", + "intValue", + "()I", + "getInt", + "putInt", + 0x1C, + 0xAC, + 3), + LONG( + long.class, + "J", + "java/lang/Long", + "(J)Ljava/lang/Long;", + "longValue", + "()J", + "getLong", + "putLong", + 0x20, + 0xAD, + 4), + FLOAT( + float.class, + "F", + "java/lang/Float", + "(F)Ljava/lang/Float;", + "floatValue", + "()F", + "getFloat", + "putFloat", + 0x24, + 0xAE, + 3), + DOUBLE( + double.class, + "D", + "java/lang/Double", + "(D)Ljava/lang/Double;", + "doubleValue", + "()D", + "getDouble", + "putDouble", + 0x28, + 0xAF, + 4); + + private final Class type; + private final String desc; + private final String wrapper; + private final String valueOfDesc; + private final String unboxName; + private final String unboxDesc; + private final String getterName; + private final String setterName; + private final int loadOpcode; + private final int returnOpcode; + private final int maxLocals; + + Primitive( + Class type, + String desc, + String wrapper, + String valueOfDesc, + String unboxName, + String unboxDesc, + String getterName, + String setterName, + int loadOpcode, + int returnOpcode, + int maxLocals) { + this.type = type; + this.desc = desc; + this.wrapper = wrapper; + this.valueOfDesc = valueOfDesc; + this.unboxName = unboxName; + this.unboxDesc = unboxDesc; + this.getterName = getterName; + this.setterName = setterName; + this.loadOpcode = loadOpcode; + this.returnOpcode = returnOpcode; + this.maxLocals = maxLocals; + } + + private static Primitive of(Class type) { + for (Primitive primitive : values()) { + if (primitive.type == type) { + return primitive; + } + } + throw new IllegalArgumentException("Not a primitive field type: " + type); + } + } +} diff --git a/java/fory-core/src/main/java25/org/apache/fory/serializer/SerializedLambdaSerializer.java b/java/fory-core/src/main/java25/org/apache/fory/serializer/SerializedLambdaSerializer.java new file mode 100644 index 0000000000..a744b99fff --- /dev/null +++ b/java/fory-core/src/main/java25/org/apache/fory/serializer/SerializedLambdaSerializer.java @@ -0,0 +1,216 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.serializer; + +import java.lang.invoke.MethodHandle; +import java.lang.invoke.SerializedLambda; +import java.lang.reflect.Method; +import org.apache.fory.context.CopyContext; +import org.apache.fory.context.ReadContext; +import org.apache.fory.context.WriteContext; +import org.apache.fory.exception.ForyException; +import org.apache.fory.memory.MemoryBuffer; +import org.apache.fory.platform.AndroidSupport; +import org.apache.fory.platform.internal._JDKAccess; +import org.apache.fory.resolver.TypeResolver; +import org.apache.fory.util.Preconditions; + +/** + * Serializer for {@link SerializedLambda}. It writes the JDK lambda payload through the public + * getter API, applies {@code readResolve} on read, and preserves unresolved {@code + * SerializedLambda} form on direct copy. + */ +@SuppressWarnings({"rawtypes", "unchecked"}) +public class SerializedLambdaSerializer extends Serializer { + static final Class SERIALIZED_LAMBDA = SerializedLambda.class; + private static final MethodHandle READ_RESOLVE_HANDLE; + private final TypeResolver typeResolver; + + static { + MethodHandle readResolveHandle = null; + if (AndroidSupport.IS_ANDROID) { + // Lambda serialization is unsupported on Android. + } else { + try { + Method readResolveMethod = JavaSerializer.getReadResolveMethod(SERIALIZED_LAMBDA); + Preconditions.checkNotNull( + readResolveMethod, "Missing readResolve for " + SERIALIZED_LAMBDA); + try { + readResolveHandle = + _JDKAccess._trustedLookup(SERIALIZED_LAMBDA).unreflect(readResolveMethod); + } catch (RuntimeException e) { + // JDK25 rejects privateLookupIn for java.lang.invoke.SerializedLambda itself. With + // java.lang.invoke opened, reflective access still exposes the JDK readResolve method + // without using Unsafe. + readResolveMethod.setAccessible(true); + readResolveHandle = java.lang.invoke.MethodHandles.lookup().unreflect(readResolveMethod); + } + } catch (IllegalAccessException | RuntimeException e) { + // Keep serializer registration available; readResolve reports the missing open if used. + } + } + READ_RESOLVE_HANDLE = readResolveHandle; + } + + public SerializedLambdaSerializer(TypeResolver typeResolver, Class cls) { + super(typeResolver.getConfig(), cls); + this.typeResolver = typeResolver; + Preconditions.checkArgument(cls == SERIALIZED_LAMBDA); + } + + @Override + public void write(WriteContext writeContext, Object value) { + throwIfAndroid(); + MemoryBuffer buffer = writeContext.getBuffer(); + SerializedLambda serializedLambda = (SerializedLambda) value; + writeContext.writeStringRef(serializedLambda.getCapturingClass()); + writeContext.writeStringRef(serializedLambda.getFunctionalInterfaceClass()); + writeContext.writeStringRef(serializedLambda.getFunctionalInterfaceMethodName()); + writeContext.writeStringRef(serializedLambda.getFunctionalInterfaceMethodSignature()); + writeContext.writeStringRef(serializedLambda.getImplClass()); + writeContext.writeStringRef(serializedLambda.getImplMethodName()); + writeContext.writeStringRef(serializedLambda.getImplMethodSignature()); + buffer.writeVarInt32(serializedLambda.getImplMethodKind()); + writeContext.writeStringRef(serializedLambda.getInstantiatedMethodType()); + int capturedArgCount = serializedLambda.getCapturedArgCount(); + buffer.writeVarUInt32Small7(capturedArgCount); + for (int i = 0; i < capturedArgCount; i++) { + writeContext.writeRef(serializedLambda.getCapturedArg(i)); + } + } + + @Override + public Object copy(CopyContext copyContext, Object value) { + throwIfAndroid(); + SerializedLambda serializedLambda = (SerializedLambda) value; + int capturedArgCount = serializedLambda.getCapturedArgCount(); + Object[] capturedArgs = new Object[capturedArgCount]; + for (int i = 0; i < capturedArgCount; i++) { + capturedArgs[i] = copyContext.copyObject(serializedLambda.getCapturedArg(i)); + } + return newSerializedLambda( + serializedLambda.getCapturingClass(), + serializedLambda.getFunctionalInterfaceClass(), + serializedLambda.getFunctionalInterfaceMethodName(), + serializedLambda.getFunctionalInterfaceMethodSignature(), + serializedLambda.getImplMethodKind(), + serializedLambda.getImplClass(), + serializedLambda.getImplMethodName(), + serializedLambda.getImplMethodSignature(), + serializedLambda.getInstantiatedMethodType(), + capturedArgs); + } + + @Override + public Object read(ReadContext readContext) { + throwIfAndroid(); + return readResolve(readUnresolved(readContext)); + } + + Object readUnresolved(ReadContext readContext) { + throwIfAndroid(); + MemoryBuffer buffer = readContext.getBuffer(); + String capturingClass = readContext.readStringRef(); + String functionalInterfaceClass = readContext.readStringRef(); + String functionalInterfaceMethodName = readContext.readStringRef(); + String functionalInterfaceMethodSignature = readContext.readStringRef(); + String implClass = readContext.readStringRef(); + String implMethodName = readContext.readStringRef(); + String implMethodSignature = readContext.readStringRef(); + int implMethodKind = buffer.readVarInt32(); + String instantiatedMethodType = readContext.readStringRef(); + int capturedArgCount = buffer.readVarUInt32Small7(); + Object[] capturedArgs = new Object[capturedArgCount]; + for (int i = 0; i < capturedArgCount; i++) { + capturedArgs[i] = readContext.readRef(); + } + return newSerializedLambda( + capturingClass, + functionalInterfaceClass, + functionalInterfaceMethodName, + functionalInterfaceMethodSignature, + implMethodKind, + implClass, + implMethodName, + implMethodSignature, + instantiatedMethodType, + capturedArgs); + } + + static Object readResolve(Object replacement) { + throwIfAndroid(); + if (READ_RESOLVE_HANDLE == null) { + throw new ForyException( + "SerializedLambda.readResolve is inaccessible. On JDK25+, deserialize lambdas only " + + "with --add-opens=java.base/java.lang.invoke=" + + "org.apache.fory.core,org.apache.fory.format."); + } + try { + return READ_RESOLVE_HANDLE.invoke(replacement); + } catch (Throwable e) { + throw new RuntimeException("Can't deserialize lambda", e); + } + } + + private static void throwIfAndroid() { + if (AndroidSupport.IS_ANDROID) { + throw new UnsupportedOperationException( + "Lambda serialization is unsupported on Android; serialize explicit data objects instead."); + } + } + + private SerializedLambda newSerializedLambda( + String capturingClass, + String functionalInterfaceClass, + String functionalInterfaceMethodName, + String functionalInterfaceMethodSignature, + int implMethodKind, + String implClass, + String implMethodName, + String implMethodSignature, + String instantiatedMethodType, + Object[] capturedArgs) { + return new SerializedLambda( + loadCapturingClass(capturingClass), + functionalInterfaceClass, + functionalInterfaceMethodName, + functionalInterfaceMethodSignature, + implMethodKind, + implClass, + implMethodName, + implMethodSignature, + instantiatedMethodType, + capturedArgs); + } + + private Class loadCapturingClass(String className) { + String binaryClassName = className.replace('/', '.'); + try { + return Class.forName(binaryClassName, false, typeResolver.getClassLoader()); + } catch (ClassNotFoundException e) { + try { + return Class.forName( + binaryClassName, false, Thread.currentThread().getContextClassLoader()); + } catch (ClassNotFoundException ex) { + throw new RuntimeException("Can't load capturing class " + binaryClassName, ex); + } + } + } +} diff --git a/java/fory-core/src/main/java25/org/apache/fory/serializer/Serializers.java b/java/fory-core/src/main/java25/org/apache/fory/serializer/Serializers.java new file mode 100644 index 0000000000..6e99a27848 --- /dev/null +++ b/java/fory-core/src/main/java25/org/apache/fory/serializer/Serializers.java @@ -0,0 +1,865 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.serializer; + +import static org.apache.fory.util.function.Functions.makeGetterFunction; + +import java.lang.invoke.MethodHandle; +import java.lang.invoke.MethodHandles; +import java.lang.invoke.MethodType; +import java.lang.reflect.Constructor; +import java.lang.reflect.Method; +import java.lang.reflect.Modifier; +import java.math.BigDecimal; +import java.math.BigInteger; +import java.net.URI; +import java.nio.charset.Charset; +import java.util.Currency; +import java.util.StringTokenizer; +import java.util.UUID; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Function; +import java.util.function.ToIntFunction; +import java.util.regex.Pattern; +import org.apache.fory.Fory; +import org.apache.fory.collection.Cache; +import org.apache.fory.collection.CacheBuilder; +import org.apache.fory.collection.Tuple2; +import org.apache.fory.config.Config; +import org.apache.fory.context.CopyContext; +import org.apache.fory.context.ReadContext; +import org.apache.fory.context.WriteContext; +import org.apache.fory.memory.MemoryBuffer; +import org.apache.fory.memory.MemoryUtils; +import org.apache.fory.meta.TypeDef; +import org.apache.fory.platform.AndroidSupport; +import org.apache.fory.platform.GraalvmSupport; +import org.apache.fory.platform.internal._JDKAccess; +import org.apache.fory.reflect.FieldAccessor; +import org.apache.fory.reflect.ReflectionUtils; +import org.apache.fory.resolver.ClassResolver; +import org.apache.fory.resolver.TypeInfo; +import org.apache.fory.resolver.TypeResolver; +import org.apache.fory.serializer.CodegenSerializer.LazyInitBeanSerializer; +import org.apache.fory.serializer.collection.ChildContainerSerializers; +import org.apache.fory.serializer.collection.CollectionSerializer; +import org.apache.fory.serializer.collection.CollectionSerializers; +import org.apache.fory.serializer.collection.MapSerializer; +import org.apache.fory.serializer.collection.MapSerializers; +import org.apache.fory.serializer.scala.SingletonCollectionSerializer; +import org.apache.fory.serializer.scala.SingletonMapSerializer; +import org.apache.fory.serializer.scala.SingletonObjectSerializer; +import org.apache.fory.util.ExceptionUtils; +import org.apache.fory.util.StringUtils; + +/** Serialization utils and common serializers. */ +@SuppressWarnings({"rawtypes", "unchecked"}) +public class Serializers { + // avoid duplicate reflect inspection and cache for graalvm support too. + private static final Cache> CTR_MAP; + + static { + if (GraalvmSupport.isGraalBuildTime()) { + CTR_MAP = CacheBuilder.newBuilder().concurrencyLevel(32).build(); + } else { + CTR_MAP = CacheBuilder.newBuilder().weakKeys().softValues().build(); + } + } + + private static final MethodType SIG1 = + MethodType.methodType(void.class, TypeResolver.class, Class.class); + private static final MethodType SIG2 = MethodType.methodType(void.class, TypeResolver.class); + private static final MethodType SIG3 = + MethodType.methodType(void.class, Config.class, Class.class); + private static final MethodType SIG4 = MethodType.methodType(void.class, Config.class); + private static final MethodType SIG5 = MethodType.methodType(void.class, Class.class); + private static final MethodType SIG6 = MethodType.methodType(void.class); + + /** + * Serializer subclass must have a constructor which take parameters of type {@link TypeResolver} + * and {@link Class}, or {@link TypeResolver}, or {@link Config} and {@link Class}, or {@link + * Config}, or {@link Class}, or no-arg constructor. + */ + public static Serializer newSerializer( + Fory fory, Class type, Class serializerClass) { + return newSerializer(fory.getTypeResolver(), type, serializerClass); + } + + /** + * Serializer subclass must have a constructor which take parameters of type {@link TypeResolver} + * and {@link Class}, or {@link TypeResolver}, or {@link Config} and {@link Class}, or {@link + * Config}, or {@link Class}, or no-arg constructor. + */ + public static Serializer newSerializer( + TypeResolver typeResolver, Class type, Class serializerClass) { + TypeInfo typeInfo = typeResolver.getTypeInfo(type, false); + Serializer serializer = typeInfo == null ? null : typeInfo.getSerializer(); + try { + return buildSerializer(typeResolver, type, serializerClass); + } catch (Throwable t) { + // Some serializer may set itself in constructor as serializer, but the + // constructor failed later. For example, some final type field doesn't + // support serialization. + typeResolver.resetSerializer(type, serializer); + if (t instanceof java.lang.reflect.InvocationTargetException && t.getCause() != null) { + ExceptionUtils.throwException(t.getCause()); + } + ExceptionUtils.throwException(t); + } + throw new IllegalStateException("unreachable"); + } + + private static Serializer buildSerializer( + TypeResolver typeResolver, Class type, Class serializerClass) { + try { + Config config = typeResolver.getConfig(); + Serializer serializer = + buildBuiltinSerializer(typeResolver, config, type, serializerClass); + if (serializer != null) { + return serializer; + } + Tuple2 ctrInfo = CTR_MAP.getIfPresent(serializerClass); + if (ctrInfo != null) { + MethodType sig = ctrInfo.f0; + MethodHandle handle = ctrInfo.f1; + if (sig.equals(SIG1)) { + return (Serializer) handle.invoke(typeResolver, type); + } else if (sig.equals(SIG2)) { + return (Serializer) handle.invoke(typeResolver); + } else if (sig.equals(SIG3)) { + return (Serializer) handle.invoke(config, type); + } else if (sig.equals(SIG4)) { + return (Serializer) handle.invoke(config); + } else if (sig.equals(SIG5)) { + return (Serializer) handle.invoke(type); + } else { + return (Serializer) handle.invoke(); + } + } + return createSerializer(typeResolver, type, serializerClass); + } catch (Throwable t) { + ExceptionUtils.throwException(t); + throw new IllegalStateException("unreachable"); + } + } + + private static Serializer buildBuiltinSerializer( + TypeResolver typeResolver, + Config config, + Class type, + Class serializerClass) { + if (serializerClass == ObjectSerializer.class) { + return new ObjectSerializer(typeResolver, type); + } + if (serializerClass == ArraySerializers.ObjectArraySerializer.class) { + return (Serializer) new ArraySerializers.ObjectArraySerializer(typeResolver, type); + } + if (serializerClass == ObjectStreamSerializer.class) { + return new ObjectStreamSerializer(typeResolver, type); + } + if (serializerClass == ExceptionSerializers.ExceptionSerializer.class) { + return new ExceptionSerializers.ExceptionSerializer(typeResolver, type); + } + if (serializerClass == ExceptionSerializers.StackTraceElementSerializer.class) { + return (Serializer) new ExceptionSerializers.StackTraceElementSerializer(config); + } + if (serializerClass == CompatibleSerializer.class) { + TypeDef typeDef = typeResolver.getTypeDef(type, true); + return new CompatibleSerializer(typeResolver, type, typeDef); + } + if (serializerClass == EnumSerializer.class) { + return (Serializer) new EnumSerializer(config, type); + } + if (serializerClass == LambdaSerializer.class) { + return new LambdaSerializer(typeResolver, type); + } + if (serializerClass == JdkProxySerializer.class) { + return new JdkProxySerializer(typeResolver, type); + } + if (serializerClass == ReplaceResolveSerializer.class) { + return new ReplaceResolveSerializer(typeResolver, type); + } + if (serializerClass == ExternalizableSerializer.class) { + return new ExternalizableSerializer(typeResolver, type); + } + if (serializerClass == LazyInitBeanSerializer.class) { + return new LazyInitBeanSerializer(typeResolver, type); + } + if (serializerClass == TimeSerializers.CalendarSerializer.class) { + return (Serializer) new TimeSerializers.CalendarSerializer(config, type); + } + if (serializerClass == TimeSerializers.ZoneIdSerializer.class) { + return (Serializer) new TimeSerializers.ZoneIdSerializer(config, type); + } + if (serializerClass == TimeSerializers.TimeZoneSerializer.class) { + return (Serializer) new TimeSerializers.TimeZoneSerializer(config, type); + } + if (serializerClass == BufferSerializers.ByteBufferSerializer.class) { + return (Serializer) new BufferSerializers.ByteBufferSerializer(typeResolver, type); + } + if (serializerClass == CharsetSerializer.class) { + return new CharsetSerializer(config, type); + } + if (serializerClass == CollectionSerializers.EnumSetSerializer.class) { + return (Serializer) new CollectionSerializers.EnumSetSerializer(typeResolver, type); + } + if (serializerClass == CollectionSerializer.class) { + return new CollectionSerializer(typeResolver, type); + } + if (serializerClass == CollectionSerializers.DefaultJavaCollectionSerializer.class) { + return new CollectionSerializers.DefaultJavaCollectionSerializer(typeResolver, type); + } + if (serializerClass == CollectionSerializers.JDKCompatibleCollectionSerializer.class) { + return new CollectionSerializers.JDKCompatibleCollectionSerializer(typeResolver, type); + } + if (serializerClass == MapSerializer.class) { + return new MapSerializer(typeResolver, type); + } + if (serializerClass == MapSerializers.DefaultJavaMapSerializer.class) { + return new MapSerializers.DefaultJavaMapSerializer(typeResolver, type); + } + if (serializerClass == MapSerializers.JDKCompatibleMapSerializer.class) { + return new MapSerializers.JDKCompatibleMapSerializer(typeResolver, type); + } + if (serializerClass == ChildContainerSerializers.ChildCollectionSerializer.class) { + return new ChildContainerSerializers.ChildCollectionSerializer(typeResolver, type); + } + if (serializerClass == ChildContainerSerializers.ChildArrayListSerializer.class) { + return new ChildContainerSerializers.ChildArrayListSerializer(typeResolver, type); + } + if (serializerClass == ChildContainerSerializers.ChildMapSerializer.class) { + return new ChildContainerSerializers.ChildMapSerializer(typeResolver, type); + } + if (serializerClass == ChildContainerSerializers.ChildSortedSetSerializer.class) { + return new ChildContainerSerializers.ChildSortedSetSerializer(typeResolver, type); + } + if (serializerClass == ChildContainerSerializers.ChildPriorityQueueSerializer.class) { + return new ChildContainerSerializers.ChildPriorityQueueSerializer(typeResolver, type); + } + if (serializerClass == ChildContainerSerializers.ChildSortedMapSerializer.class) { + return new ChildContainerSerializers.ChildSortedMapSerializer(typeResolver, type); + } + if (serializerClass == SingletonCollectionSerializer.class) { + return new SingletonCollectionSerializer(typeResolver, type); + } + if (serializerClass == SingletonMapSerializer.class) { + return new SingletonMapSerializer(typeResolver, type); + } + if (serializerClass == SingletonObjectSerializer.class) { + return new SingletonObjectSerializer(typeResolver, type); + } + return null; + } + + private static Serializer createSerializer( + TypeResolver typeResolver, Class type, Class serializerClass) { + if (AndroidSupport.IS_ANDROID) { + return createSerializerReflectively(typeResolver, type, serializerClass); + } + try { + Config config = typeResolver.getConfig(); + try { + MethodHandle ctr = findConstructor(serializerClass, SIG1); + CTR_MAP.put(serializerClass, Tuple2.of(SIG1, ctr)); + return (Serializer) ctr.invoke(typeResolver, type); + } catch (NoSuchMethodException e) { + ExceptionUtils.ignore(e); + } + try { + MethodHandle ctr = findConstructor(serializerClass, SIG2); + CTR_MAP.put(serializerClass, Tuple2.of(SIG2, ctr)); + return (Serializer) ctr.invoke(typeResolver); + } catch (NoSuchMethodException e) { + ExceptionUtils.ignore(e); + } + try { + MethodHandle ctr = findConstructor(serializerClass, SIG3); + CTR_MAP.put(serializerClass, Tuple2.of(SIG3, ctr)); + return (Serializer) ctr.invoke(config, type); + } catch (NoSuchMethodException e) { + ExceptionUtils.ignore(e); + } + try { + MethodHandle ctr = findConstructor(serializerClass, SIG4); + CTR_MAP.put(serializerClass, Tuple2.of(SIG4, ctr)); + return (Serializer) ctr.invoke(config); + } catch (NoSuchMethodException e) { + ExceptionUtils.ignore(e); + } + try { + MethodHandle ctr = findConstructor(serializerClass, SIG5); + CTR_MAP.put(serializerClass, Tuple2.of(SIG5, ctr)); + return (Serializer) ctr.invoke(type); + } catch (NoSuchMethodException e) { + ExceptionUtils.ignore(e); + } + MethodHandle ctr = ReflectionUtils.getCtrHandle(serializerClass); + CTR_MAP.put(serializerClass, Tuple2.of(SIG6, ctr)); + return (Serializer) ctr.invoke(); + } catch (Throwable t) { + ExceptionUtils.throwException(t); + throw new IllegalStateException("unreachable"); + } + } + + private static MethodHandle findConstructor(Class cls, MethodType sig) + throws NoSuchMethodException, IllegalAccessException { + if (Modifier.isPublic(cls.getModifiers())) { + try { + return MethodHandles.publicLookup().findConstructor(cls, sig); + } catch (IllegalAccessException ignored) { + // The class may be public in a non-exported package. Fall back to the private lookup path so + // named-module users can still enable access with opens. + } + } + return _JDKAccess._trustedLookup(cls).findConstructor(cls, sig); + } + + private static Serializer createSerializerReflectively( + TypeResolver typeResolver, Class type, Class serializerClass) { + Config config = typeResolver.getConfig(); + try { + Constructor ctr = + serializerClass.getDeclaredConstructor(TypeResolver.class, Class.class); + ctr.setAccessible(true); + return (Serializer) ctr.newInstance(typeResolver, type); + } catch (NoSuchMethodException e) { + ExceptionUtils.ignore(e); + } catch (Throwable t) { + ExceptionUtils.throwException(t); + throw new IllegalStateException("unreachable"); + } + try { + Constructor ctr = + serializerClass.getDeclaredConstructor(TypeResolver.class); + ctr.setAccessible(true); + return (Serializer) ctr.newInstance(typeResolver); + } catch (NoSuchMethodException e) { + ExceptionUtils.ignore(e); + } catch (Throwable t) { + ExceptionUtils.throwException(t); + throw new IllegalStateException("unreachable"); + } + try { + Constructor ctr = + serializerClass.getDeclaredConstructor(Config.class, Class.class); + ctr.setAccessible(true); + return (Serializer) ctr.newInstance(config, type); + } catch (NoSuchMethodException e) { + ExceptionUtils.ignore(e); + } catch (Throwable t) { + ExceptionUtils.throwException(t); + throw new IllegalStateException("unreachable"); + } + try { + Constructor ctr = serializerClass.getDeclaredConstructor(Config.class); + ctr.setAccessible(true); + return (Serializer) ctr.newInstance(config); + } catch (NoSuchMethodException e) { + ExceptionUtils.ignore(e); + } catch (Throwable t) { + ExceptionUtils.throwException(t); + throw new IllegalStateException("unreachable"); + } + try { + Constructor ctr = serializerClass.getDeclaredConstructor(Class.class); + ctr.setAccessible(true); + return (Serializer) ctr.newInstance(type); + } catch (NoSuchMethodException e) { + ExceptionUtils.ignore(e); + } catch (Throwable t) { + ExceptionUtils.throwException(t); + throw new IllegalStateException("unreachable"); + } + try { + Constructor ctr = serializerClass.getDeclaredConstructor(); + ctr.setAccessible(true); + return (Serializer) ctr.newInstance(); + } catch (NoSuchMethodException e) { + throw new IllegalArgumentException( + "Serializer " + + serializerClass.getName() + + " doesn't define a supported constructor for " + + type, + e); + } catch (Throwable t) { + ExceptionUtils.throwException(t); + throw new IllegalStateException("unreachable"); + } + } + + public static void write(WriteContext writeContext, Serializer serializer, T obj) { + serializer.write(writeContext, obj); + } + + public static T read(ReadContext readContext, Serializer serializer) { + return serializer.read(readContext); + } + + private static final ToIntFunction GET_CODER; + private static final Function GET_VALUE; + + static { + if (AndroidSupport.IS_ANDROID) { + GET_VALUE = null; + GET_CODER = null; + } else { + Function getValue; + ToIntFunction getCoder; + try { + getValue = (Function) makeGetterFunction(StringBuilder.class.getSuperclass(), "getValue"); + } catch (Throwable e) { + getValue = null; + } + try { + Method getCoderMethod = StringBuilder.class.getSuperclass().getDeclaredMethod("getCoder"); + getCoder = (ToIntFunction) makeGetterFunction(getCoderMethod, int.class); + } catch (NoSuchMethodException e) { + getCoder = null; + } catch (Throwable e) { + getCoder = null; + } + GET_VALUE = getValue; + GET_CODER = getCoder; + } + } + + public abstract static class AbstractStringBuilderSerializer + extends Serializer { + private final Config config; + + public AbstractStringBuilderSerializer(Config config, Class type) { + super(config, type); + this.config = config; + } + + @Override + public void write(WriteContext writeContext, T value) { + MemoryBuffer buffer = writeContext.getBuffer(); + StringSerializer stringSerializer = writeContext.getStringSerializer(); + if (config.isXlang()) { + stringSerializer.writeString(buffer, value.toString()); + return; + } + if (AndroidSupport.IS_ANDROID) { + stringSerializer.writeString(buffer, value.toString()); + return; + } + if (GET_VALUE == null) { + stringSerializer.writeString(buffer, value.toString()); + return; + } + if (GET_CODER != null) { + int coder = GET_CODER.applyAsInt(value); + byte[] v = (byte[]) GET_VALUE.apply(value); + int bytesLen = value.length(); + if (coder != 0) { + if (coder != 1) { + throw new UnsupportedOperationException("Unsupported coder " + coder); + } + bytesLen <<= 1; + } + long header = ((long) bytesLen << 2) | coder; + buffer.writeVarUInt64(header); + buffer.writeBytes(v, 0, bytesLen); + } else { + Object rawValue = GET_VALUE.apply(value); + if (!(rawValue instanceof char[])) { + stringSerializer.writeString(buffer, value.toString()); + return; + } + char[] v = (char[]) rawValue; + if (StringUtils.isLatin(v)) { + stringSerializer.writeCharsLatin1(buffer, v, value.length()); + } else { + stringSerializer.writeCharsUTF16(buffer, v, value.length()); + } + } + } + } + + public static final class StringBuilderSerializer + extends AbstractStringBuilderSerializer { + + public StringBuilderSerializer(Config config) { + super(config, StringBuilder.class); + } + + @Override + public StringBuilder copy(CopyContext copyContext, StringBuilder origin) { + return new StringBuilder(origin); + } + + @Override + public StringBuilder read(ReadContext readContext) { + return new StringBuilder(readContext.readString()); + } + } + + public static final class StringBufferSerializer + extends AbstractStringBuilderSerializer { + + public StringBufferSerializer(Config config) { + super(config, StringBuffer.class); + } + + @Override + public StringBuffer copy(CopyContext copyContext, StringBuffer origin) { + return new StringBuffer(origin); + } + + @Override + public StringBuffer read(ReadContext readContext) { + return new StringBuffer(readContext.readString()); + } + } + + public static final class StringTokenizerSerializer extends Serializer + implements Shareable { + public StringTokenizerSerializer(Config config) { + super(config, StringTokenizer.class); + } + + @Override + public void write(WriteContext writeContext, StringTokenizer value) { + checkStringTokenizerAccess(); + MemoryBuffer buffer = writeContext.getBuffer(); + writeContext.writeRef(Accessors.STR.getObject(value)); + writeContext.writeRef(Accessors.DELIMITERS.getObject(value)); + buffer.writeBoolean(Accessors.RET_DELIMS.getBoolean(value)); + buffer.writeVarInt32(Accessors.CURRENT_POSITION.getInt(value)); + buffer.writeVarInt32(Accessors.NEW_POSITION.getInt(value)); + buffer.writeBoolean(Accessors.DELIMS_CHANGED.getBoolean(value)); + } + + @Override + public StringTokenizer read(ReadContext readContext) { + checkStringTokenizerAccess(); + String str = (String) readContext.readRef(); + String delimiters = (String) readContext.readRef(); + boolean retDelims = readContext.getBuffer().readBoolean(); + StringTokenizer tokenizer = new StringTokenizer(str, delimiters, retDelims); + restoreState(readContext.getBuffer(), tokenizer); + return tokenizer; + } + + @Override + public StringTokenizer copy(CopyContext copyContext, StringTokenizer value) { + checkStringTokenizerAccess(); + StringTokenizer tokenizer = + new StringTokenizer( + (String) Accessors.STR.getObject(value), + (String) Accessors.DELIMITERS.getObject(value), + Accessors.RET_DELIMS.getBoolean(value)); + Accessors.CURRENT_POSITION.putInt(tokenizer, Accessors.CURRENT_POSITION.getInt(value)); + Accessors.NEW_POSITION.putInt(tokenizer, Accessors.NEW_POSITION.getInt(value)); + Accessors.DELIMS_CHANGED.putBoolean(tokenizer, Accessors.DELIMS_CHANGED.getBoolean(value)); + return tokenizer; + } + + private static void restoreState(MemoryBuffer buffer, StringTokenizer tokenizer) { + Accessors.CURRENT_POSITION.putInt(tokenizer, buffer.readVarInt32()); + Accessors.NEW_POSITION.putInt(tokenizer, buffer.readVarInt32()); + Accessors.DELIMS_CHANGED.putBoolean(tokenizer, buffer.readBoolean()); + } + + private static void checkStringTokenizerAccess() { + if (!MemoryUtils.JDK_COLLECTION_FIELD_ACCESS) { + throw stringTokenizerAccessError(); + } + } + + private static UnsupportedOperationException stringTokenizerAccessError() { + return new UnsupportedOperationException( + "StringTokenizer serialization requires JDK internal field access. On JDK25+, open " + + "java.base/java.util to org.apache.fory.core,org.apache.fory.format."); + } + + private static final class Accessors { + private static final FieldAccessor CURRENT_POSITION = + FieldAccessor.createAccessor( + ReflectionUtils.getField(StringTokenizer.class, "currentPosition")); + private static final FieldAccessor NEW_POSITION = + FieldAccessor.createAccessor( + ReflectionUtils.getField(StringTokenizer.class, "newPosition")); + private static final FieldAccessor STR = + FieldAccessor.createAccessor(ReflectionUtils.getField(StringTokenizer.class, "str")); + private static final FieldAccessor DELIMITERS = + FieldAccessor.createAccessor( + ReflectionUtils.getField(StringTokenizer.class, "delimiters")); + private static final FieldAccessor RET_DELIMS = + FieldAccessor.createAccessor( + ReflectionUtils.getField(StringTokenizer.class, "retDelims")); + private static final FieldAccessor DELIMS_CHANGED = + FieldAccessor.createAccessor( + ReflectionUtils.getField(StringTokenizer.class, "delimsChanged")); + } + } + + public static final class AtomicBooleanSerializer extends Serializer + implements Shareable { + + public AtomicBooleanSerializer(Config config) { + super(config, AtomicBoolean.class); + } + + @Override + public void write(WriteContext writeContext, AtomicBoolean value) { + writeContext.getBuffer().writeBoolean(value.get()); + } + + @Override + public AtomicBoolean copy(CopyContext copyContext, AtomicBoolean origin) { + return new AtomicBoolean(origin.get()); + } + + @Override + public AtomicBoolean read(ReadContext readContext) { + return new AtomicBoolean(readContext.getBuffer().readBoolean()); + } + } + + public static final class AtomicIntegerSerializer extends Serializer + implements Shareable { + + public AtomicIntegerSerializer(Config config) { + super(config, AtomicInteger.class); + } + + @Override + public void write(WriteContext writeContext, AtomicInteger value) { + writeContext.getBuffer().writeInt32(value.get()); + } + + @Override + public AtomicInteger copy(CopyContext copyContext, AtomicInteger origin) { + return new AtomicInteger(origin.get()); + } + + @Override + public AtomicInteger read(ReadContext readContext) { + return new AtomicInteger(readContext.getBuffer().readInt32()); + } + } + + public static final class AtomicLongSerializer extends Serializer + implements Shareable { + + public AtomicLongSerializer(Config config) { + super(config, AtomicLong.class); + } + + @Override + public void write(WriteContext writeContext, AtomicLong value) { + writeContext.getBuffer().writeInt64(value.get()); + } + + @Override + public AtomicLong copy(CopyContext copyContext, AtomicLong origin) { + return new AtomicLong(origin.get()); + } + + @Override + public AtomicLong read(ReadContext readContext) { + return new AtomicLong(readContext.getBuffer().readInt64()); + } + } + + public static final class AtomicReferenceSerializer extends Serializer + implements Shareable { + + public AtomicReferenceSerializer(Config config) { + super(config, AtomicReference.class); + } + + @Override + public void write(WriteContext writeContext, AtomicReference value) { + writeContext.writeRef(value.get()); + } + + @Override + public AtomicReference copy(CopyContext copyContext, AtomicReference origin) { + return new AtomicReference(copyContext.copyObject(origin.get())); + } + + @Override + public AtomicReference read(ReadContext readContext) { + return new AtomicReference(readContext.readRef()); + } + } + + public static final class CurrencySerializer extends ImmutableSerializer + implements Shareable { + public CurrencySerializer(Config config) { + super(config, Currency.class); + } + + @Override + public void write(WriteContext writeContext, Currency object) { + writeContext.writeString(object.getCurrencyCode()); + } + + @Override + public Currency read(ReadContext readContext) { + return Currency.getInstance(readContext.readString()); + } + } + + /** Serializer for {@link Charset}. */ + public static final class CharsetSerializer extends ImmutableSerializer + implements Shareable { + public CharsetSerializer(Config config, Class type) { + super(config, type); + } + + public void write(WriteContext writeContext, T object) { + writeContext.writeString(object.name()); + } + + public T read(ReadContext readContext) { + return (T) Charset.forName(readContext.readString()); + } + } + + public static final class URISerializer extends ImmutableSerializer + implements Shareable { + + public URISerializer(Config config) { + super(config, URI.class); + } + + @Override + public void write(WriteContext writeContext, final URI uri) { + writeContext.writeString(uri.toString()); + } + + @Override + public URI read(ReadContext readContext) { + return URI.create(readContext.readString()); + } + } + + public static final class RegexSerializer extends ImmutableSerializer + implements Shareable { + public RegexSerializer(Config config) { + super(config, Pattern.class); + } + + @Override + public void write(WriteContext writeContext, Pattern pattern) { + MemoryBuffer buffer = writeContext.getBuffer(); + writeContext.writeString(pattern.pattern()); + buffer.writeInt32(pattern.flags()); + } + + @Override + public Pattern read(ReadContext readContext) { + MemoryBuffer buffer = readContext.getBuffer(); + String regex = readContext.readString(); + int flags = buffer.readInt32(); + return Pattern.compile(regex, flags); + } + } + + public static final class UUIDSerializer extends ImmutableSerializer implements Shareable { + + public UUIDSerializer(Config config) { + super(config, UUID.class); + } + + @Override + public void write(WriteContext writeContext, final UUID uuid) { + MemoryBuffer buffer = writeContext.getBuffer(); + buffer.writeInt64(uuid.getMostSignificantBits()); + buffer.writeInt64(uuid.getLeastSignificantBits()); + } + + @Override + public UUID read(ReadContext readContext) { + MemoryBuffer buffer = readContext.getBuffer(); + return new UUID(buffer.readInt64(), buffer.readInt64()); + } + } + + public static final class ClassSerializer extends ImmutableSerializer + implements Shareable { + public ClassSerializer(Config config) { + super(config, Class.class); + } + + @Override + public void write(WriteContext writeContext, Class value) { + ((ClassResolver) writeContext.getTypeResolver()).writeClassInternal(writeContext, value); + } + + @Override + public Class read(ReadContext readContext) { + return ((ClassResolver) readContext.getTypeResolver()).readClassInternal(readContext); + } + } + + /** + * Serializer for empty object of type {@link Object}. Fory disabled serialization for jdk + * internal types which doesn't implement {@link java.io.Serializable} for security, but empty + * object is safe and used sometimes, so fory should support its serialization without disable + * serializable or class registration checks. + */ + // Use a separate serializer to avoid codegen for empty object. + public static final class EmptyObjectSerializer extends ImmutableSerializer + implements Shareable { + + public EmptyObjectSerializer(Config config) { + super(config, Object.class); + } + + @Override + public void write(WriteContext writeContext, Object value) {} + + @Override + public Object read(ReadContext readContext) { + return new Object(); + } + } + + public static void registerDefaultSerializers(TypeResolver resolver) { + Config config = resolver.getConfig(); + resolver.registerInternalSerializer(Class.class, new ClassSerializer(config)); + resolver.registerInternalSerializer(StringBuilder.class, new StringBuilderSerializer(config)); + resolver.registerInternalSerializer(StringBuffer.class, new StringBufferSerializer(config)); + // Keep this internal type id reserved even when JDK collection internals are not open; + // otherwise payloads written with access enabled decode later collection ids incorrectly. + resolver.registerInternalSerializer(StringTokenizer.class, new StringTokenizerSerializer(config)); + resolver.registerInternalSerializer(BigInteger.class, new BigIntegerSerializer(config)); + resolver.registerInternalSerializer(BigDecimal.class, new DecimalSerializer(config)); + resolver.registerInternalSerializer(AtomicBoolean.class, new AtomicBooleanSerializer(config)); + resolver.registerInternalSerializer(AtomicInteger.class, new AtomicIntegerSerializer(config)); + resolver.registerInternalSerializer(AtomicLong.class, new AtomicLongSerializer(config)); + resolver.registerInternalSerializer( + AtomicReference.class, new AtomicReferenceSerializer(config)); + resolver.registerInternalSerializer(Currency.class, new CurrencySerializer(config)); + resolver.registerInternalSerializer(URI.class, new URISerializer(config)); + resolver.registerInternalSerializer(Pattern.class, new RegexSerializer(config)); + resolver.registerInternalSerializer(UUID.class, new UUIDSerializer(config)); + resolver.registerInternalSerializer(Object.class, new EmptyObjectSerializer(config)); + } +} diff --git a/java/fory-core/src/main/java25/org/apache/fory/serializer/SlicedStringUtil.java b/java/fory-core/src/main/java25/org/apache/fory/serializer/SlicedStringUtil.java new file mode 100644 index 0000000000..924c16d1f6 --- /dev/null +++ b/java/fory-core/src/main/java25/org/apache/fory/serializer/SlicedStringUtil.java @@ -0,0 +1,285 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.serializer; + +import org.apache.fory.memory.LittleEndian; +import org.apache.fory.memory.MemoryBuffer; +import org.apache.fory.memory.NativeByteOrder; +import org.apache.fory.platform.UnsafeOps; +import org.apache.fory.util.MathUtils; +import org.apache.fory.util.StringEncodingUtils; +import org.apache.fory.util.StringUtils; + +final class SlicedStringUtil { + private static final byte LATIN1 = 0; + private static final byte UTF16 = 1; + private static final byte UTF8 = 2; + + private SlicedStringUtil() {} + + static void writeCharsLatin1WithOffset( + StringSerializer serializer, MemoryBuffer buffer, char[] chars, int offset, int count) { + int writerIndex = buffer.writerIndex(); + long header = ((long) count << 2) | LATIN1; + buffer.ensure(writerIndex + 5 + count); + byte[] targetArray = buffer.getHeapMemory(); + if (targetArray != null) { + final int targetIndex = buffer._unsafeHeapWriterIndex(); + int arrIndex = targetIndex; + arrIndex += LittleEndian.putVarUint36Small(targetArray, arrIndex, header); + writerIndex += arrIndex - targetIndex; + for (int i = 0; i < count; i++) { + targetArray[arrIndex + i] = (byte) chars[offset + i]; + } + } else { + writerIndex += buffer._unsafePutVarUint36Small(writerIndex, header); + final byte[] tmpArray = serializer.getByteArray(count); + for (int i = 0; i < count; i++) { + tmpArray[i] = (byte) chars[offset + i]; + } + buffer.put(writerIndex, tmpArray, 0, count); + } + writerIndex += count; + buffer._unsafeWriterIndex(writerIndex); + } + + static void writeCharsUTF16WithOffset( + StringSerializer serializer, MemoryBuffer buffer, char[] chars, int offset, int count) { + int numBytes = MathUtils.doubleExact(count); + int writerIndex = buffer.writerIndex(); + long header = ((long) numBytes << 2) | UTF16; + buffer.ensure(writerIndex + 5 + numBytes); + final byte[] targetArray = buffer.getHeapMemory(); + if (targetArray != null) { + final int targetIndex = buffer._unsafeHeapWriterIndex(); + int arrIndex = targetIndex; + arrIndex += LittleEndian.putVarUint36Small(targetArray, arrIndex, header); + writerIndex += arrIndex - targetIndex + numBytes; + if (NativeByteOrder.IS_LITTLE_ENDIAN) { + writeCharsUTF16BEToHeap(chars, offset, arrIndex, numBytes, targetArray); + } else { + writeCharsUTF16BEToHeap(chars, offset, arrIndex, numBytes, targetArray); + } + } else { + writerIndex += buffer._unsafePutVarUint36Small(writerIndex, header); + if (NativeByteOrder.IS_LITTLE_ENDIAN) { + writerIndex = + offHeapWriteCharsUTF16WithOffset( + serializer, buffer, chars, offset, writerIndex, numBytes); + } else { + writerIndex = + offHeapWriteCharsUTF16BEWithOffset( + serializer, buffer, chars, offset, writerIndex, numBytes); + } + } + buffer._unsafeWriterIndex(writerIndex); + } + + static void writeCharsUTF8WithOffset( + StringSerializer serializer, MemoryBuffer buffer, char[] chars, int offset, int count) { + int estimateMaxBytes = count * 3; + int approxNumBytes = (int) (count * 1.5) + 1; + int writerIndex = buffer.writerIndex(); + buffer.ensure(writerIndex + 9 + estimateMaxBytes); + byte[] targetArray = buffer.getHeapMemory(); + if (targetArray != null) { + int targetIndex = buffer._unsafeHeapWriterIndex(); + int headerPos = targetIndex; + int arrIndex = targetIndex; + long header = ((long) approxNumBytes << 2) | UTF8; + int headerBytesWritten = LittleEndian.putVarUint36Small(targetArray, arrIndex, header); + arrIndex += headerBytesWritten; + writerIndex += headerBytesWritten; + targetIndex = + StringEncodingUtils.convertUTF16ToUTF8(chars, offset, count, targetArray, arrIndex); + byte stashedByte = targetArray[arrIndex]; + int written = targetIndex - arrIndex; + header = ((long) written << 2) | UTF8; + int diff = + LittleEndian.putVarUint36Small(targetArray, headerPos, header) - headerBytesWritten; + if (diff != 0) { + handleWriteCharsUTF8UnalignedHeaderBytes(targetArray, arrIndex, diff, written, stashedByte); + } + buffer._unsafeWriterIndex(writerIndex + written + diff); + } else { + final byte[] tmpArray = serializer.getByteArray(estimateMaxBytes); + int written = StringEncodingUtils.convertUTF16ToUTF8(chars, offset, count, tmpArray, 0); + long header = ((long) written << 2) | UTF8; + writerIndex += buffer._unsafePutVarUint36Small(writerIndex, header); + buffer.put(writerIndex, tmpArray, 0, written); + buffer._unsafeWriterIndex(writerIndex + written); + } + } + + static void writeCharsUTF8PerfOptimizedWithOffset( + StringSerializer serializer, MemoryBuffer buffer, char[] chars, int offset, int count) { + int estimateMaxBytes = count * 3; + int numBytes = MathUtils.doubleExact(count); + int writerIndex = buffer.writerIndex(); + long header = ((long) numBytes << 2) | UTF8; + buffer.ensure(writerIndex + 9 + estimateMaxBytes); + byte[] targetArray = buffer.getHeapMemory(); + if (targetArray != null) { + int targetIndex = buffer._unsafeHeapWriterIndex(); + int arrIndex = targetIndex; + arrIndex += LittleEndian.putVarUint36Small(targetArray, arrIndex, header); + writerIndex += arrIndex - targetIndex; + targetIndex = + StringEncodingUtils.convertUTF16ToUTF8(chars, offset, count, targetArray, arrIndex + 4); + int written = targetIndex - arrIndex - 4; + buffer._unsafePutInt32(writerIndex, written); + buffer._unsafeWriterIndex(writerIndex + 4 + written); + } else { + final byte[] tmpArray = serializer.getByteArray(estimateMaxBytes); + int written = StringEncodingUtils.convertUTF16ToUTF8(chars, offset, count, tmpArray, 0); + writerIndex += buffer._unsafePutVarUint36Small(writerIndex, header); + buffer._unsafePutInt32(writerIndex, written); + writerIndex += 4; + buffer.put(writerIndex, tmpArray, 0, written); + buffer._unsafeWriterIndex(writerIndex + written); + } + } + + static boolean isLatin(char[] chars, int offset, int count) { + int end = offset + count; + int vectorizedChars = count & ~3; + int vectorEnd = offset + vectorizedChars; + long byteOffset = UnsafeOps.CHAR_ARRAY_OFFSET + ((long) offset << 1); + long endOffset = UnsafeOps.CHAR_ARRAY_OFFSET + ((long) vectorEnd << 1); + for (long off = byteOffset; off < endOffset; off += 8) { + long multiChars = UnsafeOps.getLong(chars, off); + if ((multiChars & StringUtils.MULTI_CHARS_NON_LATIN_MASK) != 0) { + return false; + } + } + for (int i = vectorEnd; i < end; i++) { + if (chars[i] > 0xFF) { + return false; + } + } + return true; + } + + static byte bestCoder(char[] chars, int offset, int count) { + int sampleNum = Math.min(64, count); + int vectorizedLen = sampleNum >> 2; + int vectorizedChars = vectorizedLen << 2; + long byteOffset = UnsafeOps.CHAR_ARRAY_OFFSET + ((long) offset << 1); + long endOffset = byteOffset + ((long) vectorizedChars << 1); + int asciiCount = 0; + int latin1Count = 0; + int charOffset = offset; + for (long off = byteOffset; off < endOffset; off += 8, charOffset += 4) { + long multiChars = UnsafeOps.getLong(chars, off); + if ((multiChars & StringUtils.MULTI_CHARS_NON_ASCII_MASK) == 0) { + latin1Count += 4; + asciiCount += 4; + } else if ((multiChars & StringUtils.MULTI_CHARS_NON_LATIN_MASK) == 0) { + latin1Count += 4; + for (int i = 0; i < 4; ++i) { + if (chars[charOffset + i] < 0x80) { + asciiCount++; + } + } + } else { + for (int i = 0; i < 4; ++i) { + char c = chars[charOffset + i]; + if (c < 0x80) { + latin1Count++; + asciiCount++; + } else if (c <= 0xFF) { + latin1Count++; + } + } + } + } + + for (int i = vectorizedChars; i < sampleNum; i++) { + char c = chars[offset + i]; + if (c < 0x80) { + latin1Count++; + asciiCount++; + } else if (c <= 0xFF) { + latin1Count++; + } + } + + if (latin1Count == count || (latin1Count == sampleNum && isLatin(chars, offset, count))) { + return LATIN1; + } else if (asciiCount >= sampleNum * 0.5) { + return UTF8; + } else { + return UTF16; + } + } + + private static void handleWriteCharsUTF8UnalignedHeaderBytes( + byte[] targetArray, int arrIndex, int diff, int written, byte stashed) { + if (diff == 1) { + System.arraycopy(targetArray, arrIndex + 1, targetArray, arrIndex + 2, written - 1); + targetArray[arrIndex + 1] = stashed; + } else { + System.arraycopy(targetArray, arrIndex, targetArray, arrIndex - 1, written); + } + } + + private static void writeCharsUTF16BEToHeap( + char[] chars, int offset, int arrIndex, int numBytes, byte[] targetArray) { + int charIndex = offset; + for (int i = arrIndex, end = i + numBytes; i < end; i += 2) { + char c = chars[charIndex++]; + targetArray[i] = (byte) c; + targetArray[i + 1] = (byte) (c >>> 8); + } + } + + private static int offHeapWriteCharsUTF16WithOffset( + StringSerializer serializer, + MemoryBuffer buffer, + char[] chars, + int offset, + int writerIndex, + int numBytes) { + byte[] tmpArray = serializer.getByteArray(numBytes); + writeCharsUTF16BEToHeap(chars, offset, 0, numBytes, tmpArray); + buffer.put(writerIndex, tmpArray, 0, numBytes); + writerIndex += numBytes; + return writerIndex; + } + + private static int offHeapWriteCharsUTF16BEWithOffset( + StringSerializer serializer, + MemoryBuffer buffer, + char[] chars, + int offset, + int writerIndex, + int numBytes) { + byte[] tmpArray = serializer.getByteArray(numBytes); + int charIndex = offset; + for (int i = 0; i < numBytes; i += 2) { + char c = chars[charIndex++]; + tmpArray[i] = (byte) c; + tmpArray[i + 1] = (byte) (c >>> 8); + } + buffer.put(writerIndex, tmpArray, 0, numBytes); + writerIndex += numBytes; + return writerIndex; + } +} diff --git a/java/fory-core/src/main/java25/org/apache/fory/serializer/StringSerializer.java b/java/fory-core/src/main/java25/org/apache/fory/serializer/StringSerializer.java new file mode 100644 index 0000000000..e290be4069 --- /dev/null +++ b/java/fory-core/src/main/java25/org/apache/fory/serializer/StringSerializer.java @@ -0,0 +1,1166 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.serializer; + +import static org.apache.fory.type.TypeUtils.STRING_TYPE; +import static org.apache.fory.util.StringUtils.MULTI_CHARS_NON_ASCII_MASK; +import static org.apache.fory.util.StringUtils.MULTI_CHARS_NON_LATIN_MASK; + +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import org.apache.fory.annotation.CodegenInvoke; +import org.apache.fory.codegen.Expression; +import org.apache.fory.codegen.Expression.Invoke; +import org.apache.fory.codegen.Expression.StaticInvoke; +import org.apache.fory.config.Config; +import org.apache.fory.context.ReadContext; +import org.apache.fory.context.WriteContext; +import org.apache.fory.memory.LittleEndian; +import org.apache.fory.memory.MemoryBuffer; +import org.apache.fory.memory.NativeByteOrder; +import org.apache.fory.platform.AndroidSupport; +import org.apache.fory.platform.UnsafeOps; +import org.apache.fory.platform.internal._JDKAccess; +import org.apache.fory.util.MathUtils; +import org.apache.fory.util.Preconditions; +import org.apache.fory.util.StringEncodingUtils; +import org.apache.fory.util.StringUtils; + +/** + * String serializer based on method handles and byte-array accessors for speed. + * + *

Note that string operations is very common in serialization, and jvm inline and branch + * elimination is not reliable even in c2 compiler, so we try to inline and avoid checks as we can + * manually. + */ +@SuppressWarnings("unchecked") +public final class StringSerializer extends ImmutableSerializer { + private static final boolean STRING_VALUE_FIELD_IS_CHARS; + private static final boolean STRING_VALUE_FIELD_IS_BYTES; + + private static final byte LATIN1 = 0; + private static final byte UTF16 = 1; + private static final byte UTF8 = 2; + private static final int DEFAULT_BUFFER_SIZE = 1024; + + private static final boolean STRING_HAS_COUNT_OFFSET; + + static { + if (!jdkInternalFieldAccess()) { + STRING_VALUE_FIELD_IS_CHARS = false; + STRING_VALUE_FIELD_IS_BYTES = false; + STRING_HAS_COUNT_OFFSET = false; + } else { + STRING_VALUE_FIELD_IS_CHARS = _JDKAccess.STRING_VALUE_FIELD_IS_CHARS; + STRING_VALUE_FIELD_IS_BYTES = _JDKAccess.STRING_VALUE_FIELD_IS_BYTES; + STRING_HAS_COUNT_OFFSET = _JDKAccess.STRING_HAS_COUNT_OFFSET; + } + } + + private static boolean jdkInternalFieldAccess() { + return !AndroidSupport.IS_ANDROID && _JDKAccess.JDK_STRING_FIELD_ACCESS; + } + + private final boolean compressString; + private final boolean writeNumUtf16BytesForUtf8Encoding; + private final boolean xlang; + + // set default length to 0, since char array and bytes array won't be used at the same time. + private static final byte[] EMPTY_BYTES_STUB = new byte[0]; + private static final char[] EMPTY_CHARS_STUB = new char[0]; + private byte[] byteArray = EMPTY_BYTES_STUB; + private int smoothByteArrayLength = DEFAULT_BUFFER_SIZE; + private char[] charArray = EMPTY_CHARS_STUB; + private int smoothCharArrayLength = DEFAULT_BUFFER_SIZE; + private byte[] byteArray2 = EMPTY_BYTES_STUB; + + public StringSerializer(Config config) { + super(config, String.class, config.trackingRef() && !config.isStringRefIgnored()); + compressString = config.compressString(); + xlang = config.isXlang(); + if (xlang) { + Preconditions.checkArgument(compressString, "compress string muse be enabled for xlang mode"); + } + writeNumUtf16BytesForUtf8Encoding = config.writeNumUtf16BytesForUtf8Encoding(); + } + + @Override + public void write(WriteContext writeContext, String value) { + writeString(writeContext.getBuffer(), value); + } + + @Override + public String read(ReadContext readContext) { + return readString(readContext.getBuffer()); + } + + public static Expression writeStringExpr( + Expression strSerializer, Expression buffer, Expression str, boolean compressString) { + if (!jdkInternalFieldAccess()) { + return new Invoke(strSerializer, "writeString", buffer, str); + } + if (STRING_VALUE_FIELD_IS_BYTES) { + if (compressString) { + return new Invoke(strSerializer, "writeCompressedBytesString", buffer, str); + } else { + return new StaticInvoke(StringSerializer.class, "writeBytesString", buffer, str); + } + } else { + if (!STRING_VALUE_FIELD_IS_CHARS) { + throw new UnsupportedOperationException(); + } + if (STRING_HAS_COUNT_OFFSET) { + if (compressString) { + return new Invoke(strSerializer, "writeCompressedCharsStringWithOffset", buffer, str); + } else { + return new Invoke(strSerializer, "writeCharsStringWithOffset", buffer, str); + } + } else { + if (compressString) { + return new Invoke(strSerializer, "writeCompressedCharsString", buffer, str); + } else { + return new Invoke(strSerializer, "writeCharsString", buffer, str); + } + } + } + } + + public static Expression readStringExpr( + Expression strSerializer, Expression buffer, boolean compressString) { + if (!jdkInternalFieldAccess()) { + return new Invoke(strSerializer, "readString", STRING_TYPE, buffer); + } + if (STRING_VALUE_FIELD_IS_BYTES) { + if (compressString) { + return new Invoke(strSerializer, "readCompressedBytesString", STRING_TYPE, buffer); + } else { + return new Invoke(strSerializer, "readBytesString", STRING_TYPE, buffer); + } + } else { + if (!STRING_VALUE_FIELD_IS_CHARS) { + throw new UnsupportedOperationException(); + } + if (compressString) { + return new Invoke(strSerializer, "readCompressedCharsString", STRING_TYPE, buffer); + } else { + return new Invoke(strSerializer, "readCharsString", STRING_TYPE, buffer); + } + } + } + + @CodegenInvoke + public String readBytesString(MemoryBuffer buffer) { + long header = buffer.readVarUint36Small(); + byte coder = (byte) (header & 0b11); + int numBytes = (int) (header >>> 2); + byte[] bytes; + if (!NativeByteOrder.IS_LITTLE_ENDIAN && coder == UTF16) { + bytes = readBytesUTF16BE(buffer, numBytes); + } else { + bytes = readBytesUnCompressedUTF16(buffer, numBytes); + } + if (coder != UTF8) { + return newBytesStringZeroCopy(coder, bytes); + } else { + return new String(bytes, 0, numBytes, StandardCharsets.UTF_8); + } + } + + @CodegenInvoke + public String readCharsString(MemoryBuffer buffer) { + long header = buffer.readVarUint36Small(); + byte coder = (byte) (header & 0b11); + int numBytes = (int) (header >>> 2); + char[] chars; + if (coder == LATIN1) { + chars = readCharsLatin1(buffer, numBytes); + } else if (coder == UTF16) { + chars = readCharsUTF16(buffer, numBytes); + } else { + throw new RuntimeException("Unknown coder type " + coder); + } + return newCharsStringZeroCopy(chars); + } + + @CodegenInvoke + public String readCompressedBytesString(MemoryBuffer buffer) { + long header = buffer.readVarUint36Small(); + byte coder = (byte) (header & 0b11); + int numBytes = (int) (header >>> 2); + if (coder == UTF8) { + byte[] data; + if (writeNumUtf16BytesForUtf8Encoding) { + data = readBytesUTF8PerfOptimized(buffer, numBytes); + } else { + if (xlang) { + return readBytesUTF8ForXlang(buffer, numBytes); + } + data = readBytesUTF8(buffer, numBytes); + } + return newBytesStringZeroCopy(UTF16, data); + } else if (coder == LATIN1) { + return newBytesStringZeroCopy(coder, readBytesUnCompressedUTF16(buffer, numBytes)); + } else if (coder == UTF16) { + byte[] bytes; + if (NativeByteOrder.IS_LITTLE_ENDIAN) { + bytes = readBytesUnCompressedUTF16(buffer, numBytes); + } else { + bytes = readBytesUTF16BE(buffer, numBytes); + } + return newBytesStringZeroCopy(coder, bytes); + } else { + throw new RuntimeException("Unknown coder type " + coder); + } + } + + // the utf8 data may can be encoded with latin1, so the read need to check whether it can be + // encoded by latin1, if true, the coder should be latin1 instead of utf16 + String readBytesUTF8ForXlang(MemoryBuffer buffer, int numBytes) { + buffer.checkReadableBytes(numBytes); + byte[] srcArray = buffer.getHeapMemory(); + + if (srcArray != null) { + int srcIndex = buffer._unsafeHeapReaderIndex(); + + // Fast path: vectorized ASCII check (8 bytes at a time) + if (StringEncodingUtils.isUTF8WithinAscii(srcArray, srcIndex, numBytes)) { + byte[] result = new byte[numBytes]; + System.arraycopy(srcArray, srcIndex, result, 0, numBytes); + buffer._increaseReaderIndexUnsafe(numBytes); + return newBytesStringZeroCopy(LATIN1, result); + } + + // Two-pass approach: scan first, then convert + boolean isLatin1 = StringEncodingUtils.isUTF8WithinLatin1(srcArray, srcIndex, numBytes); + buffer._increaseReaderIndexUnsafe(numBytes); + + if (isLatin1) { + byte[] latin1Buffer = getByteArray(numBytes); + int latin1Len = + StringEncodingUtils.convertUTF8ToLatin1(srcArray, srcIndex, numBytes, latin1Buffer); + return newBytesStringZeroCopy(LATIN1, Arrays.copyOf(latin1Buffer, latin1Len)); + } else { + byte[] utf16Buffer = getByteArray(numBytes << 1); + int utf16Len = + StringEncodingUtils.convertUTF8ToUTF16(srcArray, srcIndex, numBytes, utf16Buffer); + return newBytesStringZeroCopy(UTF16, Arrays.copyOf(utf16Buffer, utf16Len)); + } + } else { + // Off-heap path + byte[] srcBytes = getByteArray2(numBytes); + buffer.readBytes(srcBytes, 0, numBytes); + + // Fast path: vectorized ASCII check + if (StringEncodingUtils.isUTF8WithinAscii(srcBytes, 0, numBytes)) { + // Must copy to exact size since srcBytes is a reusable buffer + return newBytesStringZeroCopy(LATIN1, Arrays.copyOf(srcBytes, numBytes)); + } + + // Two-pass approach: scan first, then convert + boolean isLatin1 = StringEncodingUtils.isUTF8WithinLatin1(srcBytes, 0, numBytes); + + if (isLatin1) { + byte[] latin1Buffer = getByteArray(numBytes); + int latin1Len = + StringEncodingUtils.convertUTF8ToLatin1(srcBytes, 0, numBytes, latin1Buffer); + return newBytesStringZeroCopy(LATIN1, Arrays.copyOf(latin1Buffer, latin1Len)); + } else { + byte[] utf16Buffer = getByteArray(numBytes << 1); + int utf16Len = StringEncodingUtils.convertUTF8ToUTF16(srcBytes, 0, numBytes, utf16Buffer); + return newBytesStringZeroCopy(UTF16, Arrays.copyOf(utf16Buffer, utf16Len)); + } + } + } + + @CodegenInvoke + public String readCompressedCharsString(MemoryBuffer buffer) { + long header = buffer.readVarUint36Small(); + byte coder = (byte) (header & 0b11); + int numBytes = (int) (header >>> 2); + char[] chars; + if (coder == LATIN1) { + chars = readCharsLatin1(buffer, numBytes); + } else if (coder == UTF8) { + return writeNumUtf16BytesForUtf8Encoding + ? readCharsUTF8PerfOptimized(buffer, numBytes) + : readCharsUTF8(buffer, numBytes); + } else if (coder == UTF16) { + chars = readCharsUTF16(buffer, numBytes); + } else { + throw new RuntimeException("Unknown coder type " + coder); + } + return newCharsStringZeroCopy(chars); + } + + // Invoked by fory JIT + public void writeString(MemoryBuffer buffer, String value) { + if (!jdkInternalFieldAccess()) { + writeStringSlow(buffer, value); + return; + } + if (STRING_VALUE_FIELD_IS_BYTES) { + if (compressString) { + writeCompressedBytesString(buffer, value); + } else { + writeBytesString(buffer, value); + } + } else { + writeJava8String(buffer, value); + } + } + + private void writeJava8String(MemoryBuffer buffer, String value) { + assert STRING_VALUE_FIELD_IS_CHARS; + if (STRING_HAS_COUNT_OFFSET) { + if (compressString) { + writeCompressedCharsStringWithOffset(buffer, value); + } else { + writeCharsStringWithOffset(buffer, value); + } + } else { + if (compressString) { + writeCompressedCharsString(buffer, value); + } else { + writeCharsString(buffer, value); + } + } + } + + // Invoked by fory JIT + public String readString(MemoryBuffer buffer) { + if (!jdkInternalFieldAccess()) { + return readStringSlow(buffer); + } + if (STRING_VALUE_FIELD_IS_BYTES) { + if (compressString) { + return readCompressedBytesString(buffer); + } else { + return readBytesString(buffer); + } + } else { + assert STRING_VALUE_FIELD_IS_CHARS; + if (compressString) { + return readCompressedCharsString(buffer); + } else { + return readCharsString(buffer); + } + } + } + + private void writeStringSlow(MemoryBuffer buffer, String value) { + char[] chars = value.toCharArray(); + if (isLatin(chars)) { + writeCharsLatin1(buffer, chars, chars.length); + return; + } + if (compressString) { + byte[] utf8Bytes = value.getBytes(StandardCharsets.UTF_8); + int utf16Bytes = chars.length << 1; + if (utf8Bytes.length < utf16Bytes) { + writeStringUtf8Slow(buffer, utf8Bytes, utf16Bytes); + return; + } + } + writeCharsUTF16(buffer, chars, chars.length); + } + + private String readStringSlow(MemoryBuffer buffer) { + long header = buffer.readVarUint36Small(); + byte coder = (byte) (header & 0b11); + int numBytes = (int) (header >>> 2); + if (coder == LATIN1) { + return new String(readBytesUnCompressedUTF16(buffer, numBytes), StandardCharsets.ISO_8859_1); + } else if (coder == UTF16) { + return new String(readCharsUTF16(buffer, numBytes)); + } else if (coder == UTF8) { + int utf8Bytes = writeNumUtf16BytesForUtf8Encoding ? buffer.readInt32() : numBytes; + return new String(buffer.readBytes(utf8Bytes), StandardCharsets.UTF_8); + } else { + throw new RuntimeException("Unknown coder type " + coder); + } + } + + private void writeStringUtf8Slow(MemoryBuffer buffer, byte[] utf8Bytes, int utf16Bytes) { + int headerLength = writeNumUtf16BytesForUtf8Encoding ? utf16Bytes : utf8Bytes.length; + writeVarUint36Small(buffer, ((long) headerLength << 2) | UTF8); + if (writeNumUtf16BytesForUtf8Encoding) { + buffer.writeInt32(utf8Bytes.length); + } + buffer.writeBytes(utf8Bytes); + } + + private static void writeVarUint36Small(MemoryBuffer buffer, long value) { + int writerIndex = buffer.writerIndex(); + buffer.ensure(writerIndex + 9); + writerIndex += buffer._unsafePutVarUint36Small(writerIndex, value); + buffer._unsafeWriterIndex(writerIndex); + } + + private static boolean isLatin(char[] chars) { + for (char c : chars) { + if (c > 0xFF) { + return false; + } + } + return true; + } + + private static Object getStringValue(String value) { + return _JDKAccess.getStringValue(value); + } + + private static byte getStringCoder(String value) { + return _JDKAccess.getStringCoder(value); + } + + private static int getStringOffset(String value) { + return _JDKAccess.getStringOffset(value); + } + + private static int getStringCount(String value) { + return _JDKAccess.getStringCount(value); + } + + @CodegenInvoke + public void writeCompressedBytesString(MemoryBuffer buffer, String value) { + final byte[] bytes = (byte[]) getStringValue(value); + final byte coder = getStringCoder(value); + if (coder == LATIN1 || bestCoder(bytes) == UTF16) { + writeBytesString(buffer, coder, bytes); + } else { + if (writeNumUtf16BytesForUtf8Encoding) { + writeBytesUTF8PerfOptimized(buffer, bytes); + } else { + writeBytesUTF8(buffer, bytes); + } + } + } + + @CodegenInvoke + public void writeCompressedCharsString(MemoryBuffer buffer, String value) { + final char[] chars = (char[]) getStringValue(value); + final byte coder = bestCoder(chars); + if (coder == LATIN1) { + writeCharsLatin1(buffer, chars, chars.length); + } else if (coder == UTF8) { + if (writeNumUtf16BytesForUtf8Encoding) { + writeCharsUTF8PerfOptimized(buffer, chars); + } else { + writeCharsUTF8(buffer, chars); + } + } else { + writeCharsUTF16(buffer, chars, chars.length); + } + } + + @CodegenInvoke + public void writeCompressedCharsStringWithOffset(MemoryBuffer buffer, String value) { + final char[] chars = (char[]) getStringValue(value); + final int offset = getStringOffset(value); + final int count = getStringCount(value); + final byte coder = SlicedStringUtil.bestCoder(chars, offset, count); + if (coder == LATIN1) { + SlicedStringUtil.writeCharsLatin1WithOffset(this, buffer, chars, offset, count); + } else if (coder == UTF8) { + if (writeNumUtf16BytesForUtf8Encoding) { + SlicedStringUtil.writeCharsUTF8PerfOptimizedWithOffset(this, buffer, chars, offset, count); + } else { + SlicedStringUtil.writeCharsUTF8WithOffset(this, buffer, chars, offset, count); + } + } else { + SlicedStringUtil.writeCharsUTF16WithOffset(this, buffer, chars, offset, count); + } + } + + @CodegenInvoke + public static void writeBytesString(MemoryBuffer buffer, String value) { + byte[] bytes = (byte[]) getStringValue(value); + byte coder = getStringCoder(value); + writeBytesString(buffer, coder, bytes); + } + + public static void writeBytesString(MemoryBuffer buffer, byte coder, byte[] bytes) { + if (!NativeByteOrder.IS_LITTLE_ENDIAN && coder == UTF16) { + writeBytesStringUTF16BE(buffer, bytes); + return; + } + int bytesLen = bytes.length; + long header = ((long) bytesLen << 2) | coder; + int writerIndex = buffer.writerIndex(); + // The `ensure` ensure next operations are safe without bound checks, + // and inner heap buffer doesn't change. + buffer.ensure(writerIndex + 9 + bytesLen); // 1 byte coder + varint max 8 bytes + final byte[] targetArray = buffer.getHeapMemory(); + if (targetArray != null) { + // Some JDK11 Unsafe.copyMemory will `copyMemoryChecks`, and + // jvm doesn't eliminate well in some jdk. + final int targetIndex = buffer._unsafeHeapWriterIndex(); + int arrIndex = targetIndex; + arrIndex += LittleEndian.putVarUint36Small(targetArray, arrIndex, header); + writerIndex += arrIndex - targetIndex; + System.arraycopy(bytes, 0, targetArray, arrIndex, bytesLen); + } else { + final int headerBytes = buffer._unsafePutVarUint36Small(writerIndex, header); + writerIndex += headerBytes; + buffer.put(writerIndex, bytes, 0, bytesLen); + } + writerIndex += bytesLen; + buffer._unsafeWriterIndex(writerIndex); + } + + @CodegenInvoke + public void writeCharsString(MemoryBuffer buffer, String value) { + final char[] chars = (char[]) getStringValue(value); + if (StringUtils.isLatin(chars)) { + writeCharsLatin1(buffer, chars, chars.length); + } else { + writeCharsUTF16(buffer, chars, chars.length); + } + } + + @CodegenInvoke + public void writeCharsStringWithOffset(MemoryBuffer buffer, String value) { + final char[] chars = (char[]) getStringValue(value); + final int offset = getStringOffset(value); + final int count = getStringCount(value); + if (SlicedStringUtil.isLatin(chars, offset, count)) { + SlicedStringUtil.writeCharsLatin1WithOffset(this, buffer, chars, offset, count); + } else { + SlicedStringUtil.writeCharsUTF16WithOffset(this, buffer, chars, offset, count); + } + } + + public char[] readCharsLatin1(MemoryBuffer buffer, int numBytes) { + buffer.checkReadableBytes(numBytes); + byte[] srcArray = buffer.getHeapMemory(); + char[] chars = new char[numBytes]; + if (srcArray != null) { + int srcIndex = buffer._unsafeHeapReaderIndex(); + for (int i = 0; i < numBytes; i++) { + chars[i] = (char) (srcArray[srcIndex++] & 0xff); + } + buffer._increaseReaderIndexUnsafe(numBytes); + } else { + byte[] tmpArray = getByteArray(numBytes); + buffer.readBytes(tmpArray, 0, numBytes); + for (int i = 0; i < numBytes; i++) { + chars[i] = (char) (tmpArray[i] & 0xff); + } + } + return chars; + } + + public byte[] readBytesUTF8(MemoryBuffer buffer, int numBytes) { + byte[] tmpArray = getByteArray(numBytes << 1); + buffer.checkReadableBytes(numBytes); + int utf16NumBytes; + byte[] srcArray = buffer.getHeapMemory(); + if (srcArray != null) { + int srcIndex = buffer._unsafeHeapReaderIndex(); + utf16NumBytes = + StringEncodingUtils.convertUTF8ToUTF16(srcArray, srcIndex, numBytes, tmpArray); + buffer._increaseReaderIndexUnsafe(numBytes); + } else { + byte[] byteArray2 = getByteArray2(numBytes); + buffer.readBytes(byteArray2, 0, numBytes); + utf16NumBytes = StringEncodingUtils.convertUTF8ToUTF16(byteArray2, 0, numBytes, tmpArray); + } + return Arrays.copyOf(tmpArray, utf16NumBytes); + } + + private byte[] readBytesUTF8PerfOptimized(MemoryBuffer buffer, int numBytes) { + int udf8Bytes = buffer.readInt32(); + byte[] bytes = new byte[numBytes]; + // noinspection Duplicates + buffer.checkReadableBytes(udf8Bytes); + byte[] srcArray = buffer.getHeapMemory(); + if (srcArray != null) { + int srcIndex = buffer._unsafeHeapReaderIndex(); + int readLen = StringEncodingUtils.convertUTF8ToUTF16(srcArray, srcIndex, udf8Bytes, bytes); + assert readLen == numBytes : "Decode UTF8 to UTF16 failed"; + buffer._increaseReaderIndexUnsafe(udf8Bytes); + } else { + byte[] tmpArray = getByteArray(udf8Bytes); + buffer.readBytes(tmpArray, 0, udf8Bytes); + int readLen = StringEncodingUtils.convertUTF8ToUTF16(tmpArray, 0, udf8Bytes, bytes); + assert readLen == numBytes : "Decode UTF8 to UTF16 failed"; + } + return bytes; + } + + public byte[] readBytesUnCompressedUTF16(MemoryBuffer buffer, int numBytes) { + buffer.checkReadableBytes(numBytes); + byte[] bytes; + byte[] heapMemory = buffer.getHeapMemory(); + if (heapMemory != null) { + final int arrIndex = buffer._unsafeHeapReaderIndex(); + buffer.increaseReaderIndex(numBytes); + bytes = new byte[numBytes]; + System.arraycopy(heapMemory, arrIndex, bytes, 0, numBytes); + } else { + bytes = buffer.readBytes(numBytes); + } + return bytes; + } + + public char[] readCharsUTF16(MemoryBuffer buffer, int numBytes) { + if (NativeByteOrder.IS_LITTLE_ENDIAN) { + char[] chars = new char[numBytes >> 1]; + // FIXME JDK11 utf16 string uses little-endian order. + buffer.readChars(chars, numBytes >> 1); + return chars; + } else { + return readCharsUTF16BE(buffer, numBytes); + } + } + + public String readCharsUTF8(MemoryBuffer buffer, int numBytes) { + char[] chars = getCharArray(numBytes); + int charsLen; + buffer.checkReadableBytes(numBytes); + byte[] srcArray = buffer.getHeapMemory(); + if (srcArray != null) { + int srcIndex = buffer._unsafeHeapReaderIndex(); + charsLen = StringEncodingUtils.convertUTF8ToUTF16(srcArray, srcIndex, numBytes, chars); + buffer._increaseReaderIndexUnsafe(numBytes); + } else { + byte[] tmpArray = getByteArray(numBytes); + buffer.readBytes(tmpArray, 0, numBytes); + charsLen = StringEncodingUtils.convertUTF8ToUTF16(tmpArray, 0, numBytes, chars); + } + return new String(chars, 0, charsLen); + } + + public String readCharsUTF8PerfOptimized(MemoryBuffer buffer, int numBytes) { + int udf16Chars = numBytes >> 1; + int udf8Bytes = buffer.readInt32(); + char[] chars = new char[udf16Chars]; + // noinspection Duplicates + buffer.checkReadableBytes(udf8Bytes); + byte[] srcArray = buffer.getHeapMemory(); + if (srcArray != null) { + int srcIndex = buffer._unsafeHeapReaderIndex(); + int readLen = StringEncodingUtils.convertUTF8ToUTF16(srcArray, srcIndex, udf8Bytes, chars); + assert readLen == udf16Chars : "Decode UTF8 to UTF16 failed"; + buffer._increaseReaderIndexUnsafe(udf8Bytes); + } else { + byte[] tmpArray = getByteArray(udf8Bytes); + buffer.readBytes(tmpArray, 0, udf8Bytes); + int readLen = StringEncodingUtils.convertUTF8ToUTF16(tmpArray, 0, udf8Bytes, chars); + assert readLen == udf16Chars : "Decode UTF8 to UTF16 failed"; + } + return newCharsStringZeroCopy(chars); + } + + public void writeCharsLatin1(MemoryBuffer buffer, char[] chars, int numBytes) { + int writerIndex = buffer.writerIndex(); + long header = ((long) numBytes << 2) | LATIN1; + buffer.ensure(writerIndex + 5 + numBytes); + byte[] targetArray = buffer.getHeapMemory(); + if (targetArray != null) { + final int targetIndex = buffer._unsafeHeapWriterIndex(); + int arrIndex = targetIndex; + arrIndex += LittleEndian.putVarUint36Small(targetArray, arrIndex, header); + writerIndex += arrIndex - targetIndex; + for (int i = 0; i < numBytes; i++) { + targetArray[arrIndex + i] = (byte) chars[i]; + } + } else { + writerIndex += buffer._unsafePutVarUint36Small(writerIndex, header); + final byte[] tmpArray = getByteArray(numBytes); + for (int i = 0; i < numBytes; i++) { + tmpArray[i] = (byte) chars[i]; + } + buffer.put(writerIndex, tmpArray, 0, numBytes); + } + writerIndex += numBytes; + buffer._unsafeWriterIndex(writerIndex); + } + + public void writeCharsUTF16(MemoryBuffer buffer, char[] chars, int numChars) { + int numBytes = MathUtils.doubleExact(numChars); + int writerIndex = buffer.writerIndex(); + long header = ((long) numBytes << 2) | UTF16; + buffer.ensure(writerIndex + 5 + numBytes); + final byte[] targetArray = buffer.getHeapMemory(); + if (targetArray != null) { + final int targetIndex = buffer._unsafeHeapWriterIndex(); + int arrIndex = targetIndex; + arrIndex += LittleEndian.putVarUint36Small(targetArray, arrIndex, header); + writerIndex += arrIndex - targetIndex + numBytes; + if (NativeByteOrder.IS_LITTLE_ENDIAN) { + if (AndroidSupport.IS_ANDROID) { + writeCharsUTF16ToHeapSlow(chars, arrIndex, numBytes, targetArray); + } else { + writeCharsUTF16ToHeapSlow(chars, arrIndex, numBytes, targetArray); + } + } else { + writeCharsUTF16BEToHeap(chars, arrIndex, numBytes, targetArray); + } + } else { + writerIndex += buffer._unsafePutVarUint36Small(writerIndex, header); + if (NativeByteOrder.IS_LITTLE_ENDIAN) { + writerIndex = offHeapWriteCharsUTF16(buffer, chars, writerIndex, numBytes); + } else { + writerIndex = offHeapWriteCharsUTF16BE(buffer, chars, writerIndex, numBytes); + } + } + buffer._unsafeWriterIndex(writerIndex); + } + + public void writeCharsUTF8(MemoryBuffer buffer, char[] chars) { + int estimateMaxBytes = chars.length * 3; + // num bytes of utf8 should be smaller than utf16, otherwise we should + // utf16 instead. + // We can't use length in header since we don't know num chars in go/c++ + int approxNumBytes = (int) (chars.length * 1.5) + 1; + int writerIndex = buffer.writerIndex(); + // 9 for max bytes of header + buffer.ensure(writerIndex + 9 + estimateMaxBytes); + byte[] targetArray = buffer.getHeapMemory(); + if (targetArray != null) { + // noinspection Duplicates + int targetIndex = buffer._unsafeHeapWriterIndex(); + // keep this index in case actual num utf8 bytes need different bytes for header + int headerPos = targetIndex; + int arrIndex = targetIndex; + long header = ((long) approxNumBytes << 2) | UTF8; + int headerBytesWritten = LittleEndian.putVarUint36Small(targetArray, arrIndex, header); + arrIndex += headerBytesWritten; + writerIndex += headerBytesWritten; + // noinspection Duplicates + targetIndex = StringEncodingUtils.convertUTF16ToUTF8(chars, targetArray, arrIndex); + byte stashedByte = targetArray[arrIndex]; + int written = targetIndex - arrIndex; + header = ((long) written << 2) | UTF8; + int diff = + LittleEndian.putVarUint36Small(targetArray, headerPos, header) - headerBytesWritten; + if (diff != 0) { + handleWriteCharsUTF8UnalignedHeaderBytes(targetArray, arrIndex, diff, written, stashedByte); + } + buffer._unsafeWriterIndex(writerIndex + written + diff); + } else { + // noinspection Duplicates + final byte[] tmpArray = getByteArray(estimateMaxBytes); + int written = StringEncodingUtils.convertUTF16ToUTF8(chars, tmpArray, 0); + long header = ((long) written << 2) | UTF8; + writerIndex += buffer._unsafePutVarUint36Small(writerIndex, header); + buffer.put(writerIndex, tmpArray, 0, written); + buffer._unsafeWriterIndex(writerIndex + written); + } + } + + public void writeCharsUTF8PerfOptimized(MemoryBuffer buffer, char[] chars) { + int estimateMaxBytes = chars.length * 3; + int numBytes = MathUtils.doubleExact(chars.length); + // noinspection Duplicates + int writerIndex = buffer.writerIndex(); + long header = ((long) numBytes << 2) | UTF8; + buffer.ensure(writerIndex + 9 + estimateMaxBytes); + byte[] targetArray = buffer.getHeapMemory(); + if (targetArray != null) { + int targetIndex = buffer._unsafeHeapWriterIndex(); + int arrIndex = targetIndex; + arrIndex += LittleEndian.putVarUint36Small(targetArray, arrIndex, header); + writerIndex += arrIndex - targetIndex; + targetIndex = StringEncodingUtils.convertUTF16ToUTF8(chars, targetArray, arrIndex + 4); + int written = targetIndex - arrIndex - 4; + buffer._unsafePutInt32(writerIndex, written); + buffer._unsafeWriterIndex(writerIndex + 4 + written); + } else { + final byte[] tmpArray = getByteArray(estimateMaxBytes); + int written = StringEncodingUtils.convertUTF16ToUTF8(chars, tmpArray, 0); + writerIndex += buffer._unsafePutVarUint36Small(writerIndex, header); + buffer._unsafePutInt32(writerIndex, written); + writerIndex += 4; + buffer.put(writerIndex, tmpArray, 0, written); + buffer._unsafeWriterIndex(writerIndex + written); + } + } + + private void handleWriteCharsUTF8UnalignedHeaderBytes( + byte[] targetArray, int arrIndex, int diff, int written, byte stashed) { + if (diff == 1) { + System.arraycopy(targetArray, arrIndex + 1, targetArray, arrIndex + 2, written - 1); + targetArray[arrIndex + 1] = stashed; + } else { + System.arraycopy(targetArray, arrIndex, targetArray, arrIndex - 1, written); + } + } + + private void writeBytesUTF8(MemoryBuffer buffer, byte[] bytes) { + int numBytes = bytes.length; + int estimateMaxBytes = bytes.length / 2 * 3; + int writerIndex = buffer.writerIndex(); + buffer.ensure(writerIndex + 9 + estimateMaxBytes); + byte[] targetArray = buffer.getHeapMemory(); + if (targetArray != null) { + // noinspection Duplicates + int targetIndex = buffer._unsafeHeapWriterIndex(); + // keep this index in case actual num utf8 bytes need different bytes for header + int headerPos = targetIndex; + int arrIndex = targetIndex; + long header = ((long) numBytes << 2) | UTF8; + int headerBytesWritten = LittleEndian.putVarUint36Small(targetArray, arrIndex, header); + arrIndex += headerBytesWritten; + writerIndex += arrIndex - targetIndex; + // noinspection Duplicates + targetIndex = StringEncodingUtils.convertUTF16ToUTF8(bytes, targetArray, arrIndex); + byte stashedByte = targetArray[arrIndex]; + int written = targetIndex - arrIndex; + header = ((long) written << 2) | UTF8; + int diff = + LittleEndian.putVarUint36Small(targetArray, headerPos, header) - headerBytesWritten; + if (diff != 0) { + handleWriteCharsUTF8UnalignedHeaderBytes(targetArray, arrIndex, diff, written, stashedByte); + } + buffer._unsafeWriterIndex(writerIndex + written + diff); + } else { + // noinspection Duplicates + final byte[] tmpArray = getByteArray(estimateMaxBytes); + int written = StringEncodingUtils.convertUTF16ToUTF8(bytes, tmpArray, 0); + long header = ((long) written << 2) | UTF8; + writerIndex += buffer._unsafePutVarUint36Small(writerIndex, header); + buffer.put(writerIndex, tmpArray, 0, written); + buffer._unsafeWriterIndex(writerIndex + written); + } + } + + private void writeBytesUTF8PerfOptimized(MemoryBuffer buffer, byte[] bytes) { + int numBytes = bytes.length; + int estimateMaxBytes = bytes.length / 2 * 3; + int writerIndex = buffer.writerIndex(); + long header = ((long) numBytes << 2) | UTF8; + buffer.ensure(writerIndex + 9 + estimateMaxBytes); + byte[] targetArray = buffer.getHeapMemory(); + if (targetArray != null) { + int targetIndex = buffer._unsafeHeapWriterIndex(); + int arrIndex = targetIndex; + arrIndex += LittleEndian.putVarUint36Small(targetArray, arrIndex, header); + writerIndex += arrIndex - targetIndex; + targetIndex = StringEncodingUtils.convertUTF16ToUTF8(bytes, targetArray, arrIndex + 4); + int written = targetIndex - arrIndex - 4; + buffer._unsafePutInt32(writerIndex, written); + buffer._unsafeWriterIndex(writerIndex + 4 + written); + } else { + final byte[] tmpArray = getByteArray(estimateMaxBytes); + int written = StringEncodingUtils.convertUTF16ToUTF8(bytes, tmpArray, 0); + writerIndex += buffer._unsafePutVarUint36Small(writerIndex, header); + buffer._unsafePutInt32(writerIndex, written); + writerIndex += 4; + buffer.put(writerIndex, tmpArray, 0, written); + buffer._unsafeWriterIndex(writerIndex + written); + } + } + + public static String newCharsStringZeroCopy(char[] data) { + if (!jdkInternalFieldAccess()) { + return newCharsStringSlow(data); + } + return _JDKAccess.newCharsStringZeroCopy(data); + } + + private static String newCharsStringSlow(char[] data) { + return new String(data); + } + + // coder param first to make inline call args + // `(buffer.readByte(), buffer.readBytesWithSizeEmbedded())` work. + public static String newBytesStringZeroCopy(byte coder, byte[] data) { + if (!jdkInternalFieldAccess()) { + return newBytesStringSlow(coder, data); + } + return _JDKAccess.newBytesStringZeroCopy(coder, data); + } + + private static String newBytesStringSlow(byte coder, byte[] data) { + if (coder == LATIN1) { + return new String(data, StandardCharsets.ISO_8859_1); + } else if (coder == UTF16) { + char[] chars = new char[data.length >> 1]; + for (int i = 0, j = 0; i < data.length; i += 2) { + chars[j++] = (char) ((data[i] & 0xff) | ((data[i + 1] & 0xff) << 8)); + } + return new String(chars); + } else { + return new String(data, StandardCharsets.UTF_8); + } + } + + private static void writeCharsUTF16BEToHeap( + char[] chars, int arrIndex, int numBytes, byte[] targetArray) { + // Write to heap memory then copy is 250% faster than unsafe write to direct memory. + int charIndex = 0; + for (int i = arrIndex, end = i + numBytes; i < end; i += 2) { + char c = chars[charIndex++]; + targetArray[i] = (byte) c; + targetArray[i + 1] = (byte) (c >>> 8); + } + } + + private static void writeCharsUTF16ToHeapSlow( + char[] chars, int arrIndex, int numBytes, byte[] targetArray) { + writeCharsUTF16BEToHeap(chars, arrIndex, numBytes, targetArray); + } + + private int offHeapWriteCharsUTF16( + MemoryBuffer buffer, char[] chars, int writerIndex, int numBytes) { + byte[] tmpArray = getByteArray(numBytes); + int charIndex = 0; + for (int i = 0; i < numBytes; i += 2) { + char c = chars[charIndex++]; + tmpArray[i] = (byte) (c >> StringUTF16.HI_BYTE_SHIFT); + tmpArray[i + 1] = (byte) (c >> StringUTF16.LO_BYTE_SHIFT); + } + buffer.put(writerIndex, tmpArray, 0, numBytes); + writerIndex += numBytes; + return writerIndex; + } + + private int offHeapWriteCharsUTF16BE( + MemoryBuffer buffer, char[] chars, int writerIndex, int numBytes) { + byte[] tmpArray = getByteArray(numBytes); + int charIndex = 0; + for (int i = 0; i < numBytes; i += 2) { + char c = chars[charIndex++]; + tmpArray[i] = (byte) c; + tmpArray[i + 1] = (byte) (c >>> 8); + } + buffer.put(writerIndex, tmpArray, 0, numBytes); + writerIndex += numBytes; + return writerIndex; + } + + private char[] readCharsUTF16BE(MemoryBuffer buffer, int numBytes) { + buffer.checkReadableBytes(numBytes); + final byte[] targetArray = buffer.getHeapMemory(); + char[] chars = new char[numBytes >> 1]; + if (targetArray != null) { + int charIndex = 0; + for (int i = buffer._unsafeHeapReaderIndex(), end = i + numBytes; i < end; i += 2) { + int lo = targetArray[i] & 0xff; + int hi = targetArray[i + 1] & 0xff; + chars[charIndex++] = (char) (lo | (hi << 8)); + } + buffer._increaseReaderIndexUnsafe(numBytes); + } else { + final byte[] tmpArray = getByteArray(numBytes); + buffer.readBytes(tmpArray, 0, numBytes); + int charIndex = 0; + for (int i = 0; i < numBytes; i += 2) { + int lo = tmpArray[i] & 0xff; + int hi = tmpArray[i + 1] & 0xff; + chars[charIndex++] = (char) (lo | (hi << 8)); + } + } + return chars; + } + + private byte[] readBytesUTF16BE(MemoryBuffer buffer, int numBytes) { + byte[] bytes = readBytesUnCompressedUTF16(buffer, numBytes); + swapUTF16BytesInPlace(bytes); + return bytes; + } + + private static void swapUTF16BytesInPlace(byte[] bytes) { + for (int i = 0; i < bytes.length; i += 2) { + byte tmp = bytes[i]; + bytes[i] = bytes[i + 1]; + bytes[i + 1] = tmp; + } + } + + private static void writeBytesStringUTF16BE(MemoryBuffer buffer, byte[] bytes) { + int bytesLen = bytes.length; + long header = ((long) bytesLen << 2) | UTF16; + int writerIndex = buffer.writerIndex(); + buffer.ensure(writerIndex + 9 + bytesLen); + final byte[] targetArray = buffer.getHeapMemory(); + if (targetArray != null) { + final int targetIndex = buffer._unsafeHeapWriterIndex(); + int arrIndex = targetIndex; + arrIndex += LittleEndian.putVarUint36Small(targetArray, arrIndex, header); + writerIndex += arrIndex - targetIndex; + for (int i = 0; i < bytesLen; i += 2) { + targetArray[arrIndex + i] = bytes[i + 1]; + targetArray[arrIndex + i + 1] = bytes[i]; + } + } else { + writerIndex += buffer._unsafePutVarUint36Small(writerIndex, header); + byte[] tmpArray = new byte[bytesLen]; + for (int i = 0; i < bytesLen; i += 2) { + tmpArray[i] = bytes[i + 1]; + tmpArray[i + 1] = bytes[i]; + } + buffer.put(writerIndex, tmpArray, 0, bytesLen); + } + buffer._unsafeWriterIndex(writerIndex + bytesLen); + } + + private static byte bestCoder(char[] chars) { + int numChars = chars.length; + // sample 64 chars + int sampleNum = Math.min(64, numChars); + int vectorizedLen = sampleNum >> 2; + int vectorizedChars = vectorizedLen << 2; + int endOffset = UnsafeOps.CHAR_ARRAY_OFFSET + (vectorizedChars << 1); + int asciiCount = 0; + int latin1Count = 0; + for (int offset = UnsafeOps.CHAR_ARRAY_OFFSET, charOffset = 0; + offset < endOffset; + offset += 8, charOffset += 4) { + long multiChars = UnsafeOps.getLong(chars, offset); + if ((multiChars & MULTI_CHARS_NON_ASCII_MASK) == 0) { + latin1Count += 4; + asciiCount += 4; + } else if ((multiChars & MULTI_CHARS_NON_LATIN_MASK) == 0) { + latin1Count += 4; + for (int i = 0; i < 4; ++i) { + if (chars[charOffset + i] < 0x80) { + asciiCount++; + } + } + } else { + for (int i = 0; i < 4; ++i) { + if (chars[charOffset + i] < 0x80) { + latin1Count++; + asciiCount++; + } else if (chars[charOffset + i] <= 0xFF) { + latin1Count++; + } + } + } + } + + for (int i = vectorizedChars; i < sampleNum; i++) { + if (chars[i] < 0x80) { + latin1Count++; + asciiCount++; + } else if (chars[i] <= 0xFF) { + latin1Count++; + } + } + + if (latin1Count == numChars + || (latin1Count == sampleNum && StringUtils.isLatin(chars, sampleNum))) { + return LATIN1; + } else if (asciiCount >= sampleNum * 0.5) { + // ascii number > 50%, choose UTF-8 + return UTF8; + } else { + return UTF16; + } + } + + private static byte bestCoder(byte[] bytes) { + int numBytes = bytes.length; + // sample 64 chars + int sampleNum = Math.min(64 << 1, numBytes); + int vectorizedLen = sampleNum >> 3; + int vectorizedBytes = vectorizedLen << 3; + int endOffset = UnsafeOps.BYTE_ARRAY_OFFSET + vectorizedBytes; + int asciiCount = 0; + for (int offset = UnsafeOps.BYTE_ARRAY_OFFSET, bytesOffset = 0; + offset < endOffset; + offset += 8, bytesOffset += 8) { + long multiChars = UnsafeOps.getLong(bytes, offset); + if ((multiChars & MULTI_CHARS_NON_ASCII_MASK) == 0) { + asciiCount += 4; + } else { + for (int i = 0; i < 8; i += 2) { + if (UnsafeOps.getChar(bytes, offset + i) < 0x80) { + asciiCount++; + } + } + } + } + for (int i = vectorizedBytes; vectorizedBytes < sampleNum; vectorizedBytes += 2) { + if (UnsafeOps.getChar(bytes, UnsafeOps.BYTE_ARRAY_OFFSET + i) < 0x80) { + asciiCount++; + } + } + // ascii number > 50%, choose UTF-8 + if (asciiCount >= sampleNum * 0.5) { + return UTF8; + } else { + return UTF16; + } + } + + private char[] getCharArray(int numElements) { + char[] charArray = this.charArray; + if (charArray.length < numElements) { + charArray = new char[numElements]; + this.charArray = charArray; + } + if (charArray.length > DEFAULT_BUFFER_SIZE) { + smoothCharArrayLength = + Math.max(((int) (smoothCharArrayLength * 0.9 + numElements * 0.1)), DEFAULT_BUFFER_SIZE); + if (smoothByteArrayLength <= DEFAULT_BUFFER_SIZE) { + this.charArray = new char[DEFAULT_BUFFER_SIZE]; + } + } + return charArray; + } + + byte[] getByteArray(int numElements) { + byte[] byteArray = this.byteArray; + if (byteArray.length < numElements) { + byteArray = new byte[numElements]; + this.byteArray = byteArray; + } + if (byteArray.length > DEFAULT_BUFFER_SIZE) { + smoothByteArrayLength = + Math.max(((int) (smoothByteArrayLength * 0.9 + numElements * 0.1)), DEFAULT_BUFFER_SIZE); + if (smoothByteArrayLength <= DEFAULT_BUFFER_SIZE) { + this.byteArray = new byte[DEFAULT_BUFFER_SIZE]; + } + } + return byteArray; + } + + private byte[] getByteArray2(int numElements) { + byte[] byteArray2 = this.byteArray2; + if (byteArray2.length < numElements) { + byteArray2 = new byte[numElements]; + this.byteArray = byteArray2; + } + if (byteArray2.length > DEFAULT_BUFFER_SIZE) { + smoothByteArrayLength = + Math.max(((int) (smoothByteArrayLength * 0.9 + numElements * 0.1)), DEFAULT_BUFFER_SIZE); + if (smoothByteArrayLength <= DEFAULT_BUFFER_SIZE) { + this.byteArray2 = new byte[DEFAULT_BUFFER_SIZE]; + } + } + return byteArray2; + } + + public void clearBuffer(int size) { + if (byteArray.length >= size) { + byteArray = EMPTY_BYTES_STUB; + } + if (byteArray2.length >= size) { + byteArray2 = EMPTY_BYTES_STUB; + } + if (charArray.length >= size) { + this.charArray = EMPTY_CHARS_STUB; + } + } +} diff --git a/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties b/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties index 9ffc90564f..2995057ccd 100644 --- a/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties +++ b/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties @@ -251,6 +251,7 @@ Args=--initialize-at-build-time=org.apache.fory.annotation.ForyField$Dynamic,\ org.apache.fory.reflect.TypeRef$TypeVariableKey,\ org.apache.fory.reflect.TypeRef,\ org.apache.fory.reflect.ObjectCreators,\ + org.apache.fory.reflect.ObjectCreators$ConstructorObjectCreator,\ org.apache.fory.reflect.ObjectCreators$UnsafeObjectCreator,\ org.apache.fory.reflect.ObjectCreators$DeclaredNoArgCtrObjectCreator,\ org.apache.fory.reflect.ObjectCreators$ParentNoArgCtrObjectCreator,\ @@ -331,11 +332,12 @@ Args=--initialize-at-build-time=org.apache.fory.annotation.ForyField$Dynamic,\ org.apache.fory.serializer.BufferSerializers$ByteBufferSerializer,\ org.apache.fory.serializer.CompatibleSerializer,\ org.apache.fory.serializer.CompatibleLayerSerializer,\ + org.apache.fory.serializer.CompatibleLayerSerializerBase$FieldOnlyCreator,\ org.apache.fory.serializer.EnumSerializer,\ org.apache.fory.serializer.ExceptionSerializers,\ org.apache.fory.serializer.ExceptionSerializers$ExceptionSerializer,\ + org.apache.fory.serializer.ExceptionSerializers$FieldOnlyCreator,\ org.apache.fory.serializer.ExceptionSerializers$StackTraceElementSerializer,\ - org.apache.fory.serializer.ExceptionSerializers$ThrowableOffsets,\ org.apache.fory.serializer.collection.SubListSerializers,\ org.apache.fory.serializer.collection.SubListSerializers$SubListSerializer,\ org.apache.fory.serializer.collection.SubListSerializers$ViewFields,\ @@ -396,7 +398,6 @@ Args=--initialize-at-build-time=org.apache.fory.annotation.ForyField$Dynamic,\ org.apache.fory.serializer.Serializers$URISerializer,\ org.apache.fory.serializer.Serializers$UUIDSerializer,\ org.apache.fory.serializer.Serializers,\ - org.apache.fory.serializer.StringSerializer$Offset,\ org.apache.fory.serializer.StringSerializer,\ org.apache.fory.serializer.TimeSerializers$DateSerializer,\ org.apache.fory.serializer.TimeSerializers$DurationSerializer,\ @@ -458,11 +459,9 @@ Args=--initialize-at-build-time=org.apache.fory.annotation.ForyField$Dynamic,\ org.apache.fory.serializer.collection.MapSerializers$SingletonMapSerializer,\ org.apache.fory.serializer.collection.MapSerializers$SortedMapSerializer,\ org.apache.fory.serializer.collection.MapSerializers$XlangMapSerializer,\ - org.apache.fory.serializer.collection.SynchronizedSerializers$Offset,\ org.apache.fory.serializer.collection.SynchronizedSerializers$SynchronizedCollectionSerializer,\ org.apache.fory.serializer.collection.SynchronizedSerializers$SynchronizedMapSerializer,\ org.apache.fory.serializer.collection.SynchronizedSerializers,\ - org.apache.fory.serializer.collection.UnmodifiableSerializers$Offset,\ org.apache.fory.serializer.collection.UnmodifiableSerializers$UnmodifiableCollectionSerializer,\ org.apache.fory.serializer.collection.UnmodifiableSerializers$UnmodifiableMapSerializer,\ org.apache.fory.serializer.collection.UnmodifiableSerializers,\ @@ -492,9 +491,10 @@ Args=--initialize-at-build-time=org.apache.fory.annotation.ForyField$Dynamic,\ org.apache.fory.serializer.ObjectStreamSerializer$1,\ org.apache.fory.serializer.ObjectStreamSerializer$ForyObjectInputStream,\ org.apache.fory.serializer.ObjectStreamSerializer$ForyObjectOutputStream,\ - org.apache.fory.serializer.ObjectStreamSerializer$StreamTypeInfo,\ + org.apache.fory.serializer.ObjectStreamSerializer$StreamTypeInfo,\ org.apache.fory.serializer.ObjectStreamSerializer$SlotsInfo,\ org.apache.fory.serializer.collection.ChildContainerSerializers$ChildCollectionSerializer,\ + org.apache.fory.serializer.collection.ChildContainerSerializers$FieldOnlyCreator,\ org.apache.fory.serializer.collection.ChildContainerSerializers$ChildMapSerializer,\ org.apache.fory.builder.LayerMarkerClassGenerator,\ org.apache.fory.builder.LayerMarkerClassGenerator$1,\ @@ -534,7 +534,8 @@ Args=--initialize-at-build-time=org.apache.fory.annotation.ForyField$Dynamic,\ org.apache.fory.util.record.RecordUtils$2,\ org.apache.fory.util.record.RecordUtils$3,\ org.apache.fory.util.record.RecordUtils,\ - org.apache.fory.platform.internal._JDKAccess$1,\ + org.apache.fory.platform.internal._JDKAccess$ByteArrayStreamFields,\ + org.apache.fory.platform.internal._JDKAccess$StringCoderField,\ org.apache.fory.platform.internal._JDKAccess,\ org.apache.fory.platform.internal._Lookup,\ org.apache.fory.codegen.JaninoUtils$CodeStats,\ diff --git a/java/fory-core/src/test/java/org/apache/fory/ForyTest.java b/java/fory-core/src/test/java/org/apache/fory/ForyTest.java index 5a3041677f..12684e9326 100644 --- a/java/fory-core/src/test/java/org/apache/fory/ForyTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/ForyTest.java @@ -27,6 +27,7 @@ import com.google.common.collect.HashBasedTable; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import java.beans.ConstructorProperties; import java.io.Serializable; import java.lang.invoke.MethodHandles; import java.math.BigDecimal; @@ -50,7 +51,6 @@ import java.util.TreeSet; import java.util.UUID; import java.util.WeakHashMap; -import lombok.AllArgsConstructor; import lombok.Data; import lombok.EqualsAndHashCode; import org.apache.fory.annotation.Expose; @@ -434,11 +434,22 @@ class A {} } @Data - @AllArgsConstructor private static class IgnoreFields { @Ignore int f1; @Ignore long f2; long f3; + + @ConstructorProperties({"f1", "f2", "f3"}) + IgnoreFields(int f1, long f2, long f3) { + this.f1 = f1; + this.f2 = f2; + this.f3 = f3; + } + + @ConstructorProperties({"f3"}) + IgnoreFields(long f3) { + this.f3 = f3; + } } @Test @@ -451,13 +462,33 @@ public void testIgnoreFields() { } @Data - @AllArgsConstructor private static class ExposeFields { @Expose int f1; @Expose long f2; long f3; @Expose ImmutableMap map1; ImmutableMap map2; + + @ConstructorProperties({"f1", "f2", "f3", "map1", "map2"}) + ExposeFields( + int f1, + long f2, + long f3, + ImmutableMap map1, + ImmutableMap map2) { + this.f1 = f1; + this.f2 = f2; + this.f3 = f3; + this.map1 = map1; + this.map2 = map2; + } + + @ConstructorProperties({"f1", "f2", "map1"}) + ExposeFields(int f1, long f2, ImmutableMap map1) { + this.f1 = f1; + this.f2 = f2; + this.map1 = map1; + } } @Test @@ -474,11 +505,17 @@ public void testExposeFields() { } @Data - @AllArgsConstructor private static class ExposeFields2 { @Expose int f1; @Ignore long f2; long f3; + + @ConstructorProperties({"f1", "f2", "f3"}) + ExposeFields2(int f1, long f2, long f3) { + this.f1 = f1; + this.f2 = f2; + this.f3 = f3; + } } @Test @@ -596,7 +633,7 @@ public void testPkgAccessLevelParentClass() { .build(); HashBasedTable table = HashBasedTable.create(2, 4); table.put("r", "c", 100); - serDeCheckSerializer(fory, table, "Codec"); + serDeCheckSerializer(fory, table, "HashBasedTableSerializer"); } @Data @@ -674,6 +711,7 @@ static class Struct1 { int f1; String f2; + @ConstructorProperties({"f1", "f2"}) public Struct1(int f1, String f2) { this.f1 = f1; this.f2 = f2; @@ -732,10 +770,15 @@ public void testMaxDepth() { assertThrows(InsecureException.class, () -> fory.deserialize(bytes)); } - @AllArgsConstructor static class MaxDepth { int f1; Object f2; + + @ConstructorProperties({"f1", "f2"}) + MaxDepth(int f1, Object f2) { + this.f1 = f1; + this.f2 = f2; + } } @Test diff --git a/java/fory-core/src/test/java/org/apache/fory/codegen/pkgprivate/PackagePrivateMapKeyTest.java b/java/fory-core/src/test/java/org/apache/fory/codegen/pkgprivate/PackagePrivateMapKeyTest.java index 62dd2bc6ea..e13bbff62c 100644 --- a/java/fory-core/src/test/java/org/apache/fory/codegen/pkgprivate/PackagePrivateMapKeyTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/codegen/pkgprivate/PackagePrivateMapKeyTest.java @@ -21,11 +21,11 @@ import static org.testng.Assert.assertEquals; +import java.beans.ConstructorProperties; import java.io.Serializable; import java.util.EnumMap; import java.util.HashMap; import java.util.HashSet; -import java.util.LinkedHashSet; import java.util.Map; import java.util.Set; import org.apache.fory.Fory; @@ -48,7 +48,6 @@ public void testCodegenForMapWithPackagePrivateEnumKey() { ReproNode parent = new ReproNode(ReproType.TYPE_A, "p1"); ReproNode child = new ReproNode(ReproType.TYPE_B, "c1"); parent.children.add(child); - child.parents.computeIfAbsent(parent.type, k -> new LinkedHashSet<>()).add(parent); container.nodes.computeIfAbsent(ReproType.TYPE_A, k -> new HashMap<>()).put("p1", parent); container.nodes.computeIfAbsent(ReproType.TYPE_B, k -> new HashMap<>()).put("c1", child); @@ -68,20 +67,34 @@ enum ReproType implements Serializable { class ReproNode implements Serializable { final ReproType type; final String id; - final Set children = new HashSet<>(); - final Map> parents = new EnumMap<>(ReproType.class); + Set children; + Map> parents; + @ConstructorProperties({"type", "id"}) ReproNode(ReproType type, String id) { + this(type, id, new HashSet<>(), new EnumMap<>(ReproType.class)); + } + + ReproNode( + ReproType type, String id, Set children, Map> parents) { this.type = type; this.id = id; + this.children = children; + this.parents = parents; } } class ReproContainer implements Serializable { - final Map> nodes = new EnumMap<>(ReproType.class); + final Map> nodes; final String version; ReproContainer(String version) { + this(new EnumMap<>(ReproType.class), version); + } + + @ConstructorProperties({"nodes", "version"}) + ReproContainer(Map> nodes, String version) { + this.nodes = nodes; this.version = version; } } diff --git a/java/fory-core/src/test/java/org/apache/fory/memory/MemoryBufferTest.java b/java/fory-core/src/test/java/org/apache/fory/memory/MemoryBufferTest.java index 03f38a9a16..8eced95b1e 100644 --- a/java/fory-core/src/test/java/org/apache/fory/memory/MemoryBufferTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/memory/MemoryBufferTest.java @@ -32,7 +32,9 @@ import java.nio.charset.StandardCharsets; import java.util.Random; import org.apache.fory.platform.AndroidSupport; +import org.apache.fory.platform.JdkVersion; import org.testng.Assert; +import org.testng.SkipException; import org.testng.annotations.Test; public class MemoryBufferTest { @@ -84,7 +86,7 @@ public void testBufferWrite() { @Test public void testByteArrayStreamWrap() { - if (!MemoryUtils.JDK_INTERNAL_FIELD_ACCESS) { + if (!MemoryUtils.JDK_BYTE_ARRAY_STREAM_FIELD_ACCESS) { return; } ByteArrayOutputStream outputStream = new ByteArrayOutputStream(8); @@ -105,6 +107,13 @@ public void testByteArrayStreamWrap() { assertEquals(buffer.readByte(), (byte) 6); } + @Test + public void testFromDirectByteBufferRejectsHeapBuffer() { + assertThrows( + IllegalArgumentException.class, + () -> MemoryBuffer.fromDirectByteBuffer(ByteBuffer.allocate(8), 8, null)); + } + @Test public void testAndroidHeapMemoryBufferPaths() throws Exception { String javaBin = @@ -326,6 +335,17 @@ public void testWrapBuffer() { } } + @Test + public void testJdk25DirectBufferNoRawAddress() { + if (JdkVersion.MAJOR_VERSION < 25) { + throw new SkipException("Skip on jdk" + JdkVersion.MAJOR_VERSION); + } + MemoryBuffer buffer = MemoryUtils.wrap(ByteBuffer.allocateDirect(8)); + buffer.writeByte((byte) 1); + assertThrows(UnsupportedOperationException.class, () -> buffer.getUnsafeReaderAddress()); + assertThrows(UnsupportedOperationException.class, () -> buffer._unsafeWriterAddress()); + } + @Test public void testSliceAsByteBuffer() { byte[] data = new byte[10]; @@ -378,6 +398,48 @@ public void testEqualToZeroSize() { Assert.assertTrue(buf1.equalTo(buf2, 0, 0, buf1.size())); } + @Test + public void testDirectCopyTo() { + byte[] values = new byte[16]; + for (int i = 0; i < values.length; i++) { + values[i] = (byte) i; + } + MemoryBuffer source = MemoryUtils.wrap(ByteBuffer.allocateDirect(values.length)); + source.writeBytes(values); + MemoryBuffer directTarget = MemoryUtils.wrap(ByteBuffer.allocateDirect(values.length)); + source.copyTo(0, directTarget, 0, values.length); + assertEquals(directTarget.getBytes(0, values.length), values); + + MemoryBuffer heapTarget = MemoryUtils.buffer(values.length); + source.copyTo(0, heapTarget, 0, values.length); + assertEquals(heapTarget.getBytes(0, values.length), values); + + MemoryBuffer heapSource = MemoryUtils.wrap(values); + MemoryBuffer directFromHeap = MemoryUtils.wrap(ByteBuffer.allocateDirect(values.length)); + heapSource.copyTo(0, directFromHeap, 0, values.length); + assertEquals(directFromHeap.getBytes(0, values.length), values); + + source.copyTo(0, source, 4, 8); + assertEquals(source.getBytes(4, 8), new byte[] {0, 1, 2, 3, 4, 5, 6, 7}); + } + + @Test + public void testDirectPrimitiveArrays() { + MemoryBuffer direct = MemoryUtils.wrap(ByteBuffer.allocateDirect(64)); + int[] ints = {1, -2, 3, Integer.MIN_VALUE}; + long[] longs = {4L, -5L, Long.MAX_VALUE}; + direct.writeInts(ints); + direct.writeLongs(longs); + + int[] readInts = new int[ints.length]; + long[] readLongs = new long[longs.length]; + direct.readerIndex(0); + direct.readInts(readInts, 0, readInts.length); + direct.readLongs(readLongs, 0, readLongs.length); + assertEquals(readInts, ints); + assertEquals(readLongs, longs); + } + @Test public void testWritePrimitiveArrayWithSizeEmbedded() { MemoryBuffer buf = MemoryUtils.buffer(16); diff --git a/java/fory-core/src/test/java/org/apache/fory/meta/TypeDefTest.java b/java/fory-core/src/test/java/org/apache/fory/meta/TypeDefTest.java index 6367a8625c..0b79050727 100644 --- a/java/fory-core/src/test/java/org/apache/fory/meta/TypeDefTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/meta/TypeDefTest.java @@ -25,13 +25,10 @@ import com.google.common.collect.ImmutableList; import java.lang.reflect.Field; -import java.util.ArrayList; import java.util.Collection; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.TreeSet; import org.apache.fory.Fory; import org.apache.fory.ForyTestBase; import org.apache.fory.annotation.ForyField; diff --git a/java/fory-core/src/test/java/org/apache/fory/reflect/FieldAccessorTest.java b/java/fory-core/src/test/java/org/apache/fory/reflect/FieldAccessorTest.java index 68209013fb..c86faa6bf5 100644 --- a/java/fory-core/src/test/java/org/apache/fory/reflect/FieldAccessorTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/reflect/FieldAccessorTest.java @@ -27,6 +27,7 @@ import java.nio.charset.StandardCharsets; import lombok.AllArgsConstructor; import org.apache.fory.platform.AndroidSupport; +import org.apache.fory.platform.JdkVersion; import org.apache.fory.reflect.FieldAccessor.GeneratedAccessor; import org.testng.Assert; import org.testng.annotations.Test; @@ -65,6 +66,34 @@ public void testGeneratedAccessor() throws Exception { Assert.assertEquals(f3.getObject(struct), "b"); } + @Test + public void testHiddenAccessor() throws Exception { + HiddenFields fields = new HiddenFields(); + FieldAccessor intAccessor = + FieldAccessor.createAccessor(HiddenFields.class.getDeclaredField("i")); + Assert.assertEquals(intAccessor.getInt(fields), 1); + intAccessor.putInt(fields, 2); + Assert.assertEquals(intAccessor.getInt(fields), 2); + + FieldAccessor objectAccessor = + FieldAccessor.createAccessor(HiddenFields.class.getDeclaredField("text")); + Assert.assertEquals(objectAccessor.getObject(fields), "a"); + objectAccessor.putObject(fields, "b"); + Assert.assertEquals(objectAccessor.getObject(fields), "b"); + + FieldAccessor finalAccessor = + FieldAccessor.createAccessor(HiddenFields.class.getDeclaredField("finalValue")); + Assert.assertEquals(finalAccessor.getLong(fields), 3L); + if (JdkVersion.MAJOR_VERSION >= 25) { + Assert.assertTrue(isHidden(intAccessor.getClass())); + Assert.assertTrue(isHidden(objectAccessor.getClass())); + } + } + + private static boolean isHidden(Class cls) throws Exception { + return (Boolean) Class.class.getMethod("isHidden").invoke(cls); + } + @Test public void testAndroidReflectionFieldAccessorPaths() throws Exception { String javaBin = @@ -145,4 +174,10 @@ private static final class AndroidFields { private double doubleValue = 3.5d; private Object objectValue = "before"; } + + private static final class HiddenFields { + private int i = 1; + private String text = "a"; + private final long finalValue = 3; + } } diff --git a/java/fory-core/src/test/java/org/apache/fory/resolver/DisallowedListTest.java b/java/fory-core/src/test/java/org/apache/fory/resolver/DisallowedListTest.java index f30ede9db6..6c90705cb7 100644 --- a/java/fory-core/src/test/java/org/apache/fory/resolver/DisallowedListTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/resolver/DisallowedListTest.java @@ -19,12 +19,12 @@ package org.apache.fory.resolver; +import java.beans.Expression; import java.rmi.server.UnicastRemoteObject; import java.util.Set; import org.apache.fory.Fory; import org.apache.fory.ForyTestBase; import org.apache.fory.exception.InsecureException; -import org.apache.fory.platform.UnsafeOps; import org.testng.Assert; import org.testng.annotations.Test; @@ -85,7 +85,7 @@ public void testSerializeDisallowedClass() { .build(); if (requireClassRegistration) { // Registered or unregistered Classes should be subject to disallowed list restrictions. - fory.register(UnicastRemoteObject.class); + fory.register(Expression.class); } allFory[i] = fory; } @@ -93,7 +93,7 @@ public void testSerializeDisallowedClass() { for (Fory fory : allFory) { Assert.assertThrows( InsecureException.class, - () -> fory.serialize(UnsafeOps.newInstance(UnicastRemoteObject.class))); + () -> fory.serialize(new Expression(System.class, "exit", new Object[] {0}))); serDe(fory, new String[] {"a", "b"}); } } diff --git a/java/fory-core/src/test/java/org/apache/fory/serializer/ArraySerializersTest.java b/java/fory-core/src/test/java/org/apache/fory/serializer/ArraySerializersTest.java index 652a98e7c9..56d19e4291 100644 --- a/java/fory-core/src/test/java/org/apache/fory/serializer/ArraySerializersTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/serializer/ArraySerializersTest.java @@ -23,6 +23,7 @@ import static org.testng.Assert.assertThrows; import static org.testng.Assert.assertTrue; +import java.beans.ConstructorProperties; import java.lang.reflect.Array; import java.lang.reflect.Field; import java.util.Arrays; @@ -486,6 +487,7 @@ public void testArrayStructZeroCopy(Fory fory) { static class A { final int f1; + @ConstructorProperties({"f1"}) A(int f1) { this.f1 = f1; } @@ -495,6 +497,7 @@ static class A { static class B extends A { final String f2; + @ConstructorProperties({"f1", "f2"}) B(int f1, String f2) { super(f1); this.f2 = f2; @@ -517,6 +520,11 @@ static class GenericArrayWrapper { public GenericArrayWrapper(Class clazz, int capacity) { this.array = (T[]) Array.newInstance(clazz, capacity); } + + @ConstructorProperties({"array"}) + public GenericArrayWrapper(T[] array) { + this.array = array; + } } @SuppressWarnings("unchecked") diff --git a/java/fory-core/src/test/java/org/apache/fory/serializer/DuplicateFieldsTest.java b/java/fory-core/src/test/java/org/apache/fory/serializer/DuplicateFieldsTest.java index e9f665d5f9..073a7dcc87 100644 --- a/java/fory-core/src/test/java/org/apache/fory/serializer/DuplicateFieldsTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/serializer/DuplicateFieldsTest.java @@ -25,6 +25,7 @@ import lombok.ToString; import org.apache.fory.Fory; import org.apache.fory.ForyTestBase; +import org.apache.fory.annotation.ForyField; import org.apache.fory.builder.CodecUtils; import org.apache.fory.config.ForyBuilder; import org.apache.fory.memory.MemoryBuffer; @@ -46,6 +47,61 @@ public static class C extends B { int f1; } + public static class PrivateBase { + @ForyField(id = 1) + private int value; + + @ForyField(id = 2) + private final long finalValue; + + public PrivateBase() { + this(0, 0); + } + + public PrivateBase(@ForyField(id = 1) int value, @ForyField(id = 2) long finalValue) { + this.value = value; + this.finalValue = finalValue; + } + + int baseValue() { + return value; + } + + long baseFinalValue() { + return finalValue; + } + } + + public static class PrivateChild extends PrivateBase { + @ForyField(id = 3) + private int value; + + @ForyField(id = 4) + private final long finalValue; + + public PrivateChild() { + this(0, 0, 0, 0); + } + + public PrivateChild( + @ForyField(id = 1) int baseValue, + @ForyField(id = 2) long baseFinalValue, + @ForyField(id = 3) int value, + @ForyField(id = 4) long finalValue) { + super(baseValue, baseFinalValue); + this.value = value; + this.finalValue = finalValue; + } + + int childValue() { + return value; + } + + long childFinalValue() { + return finalValue; + } + } + @Test() public void testDuplicateFieldsNoCompatible() { C c = new C(); @@ -96,6 +152,30 @@ public void testDuplicateFieldsNoCompatible() { } } + @Test + public void testPrivateDuplicateFieldsNoCompatible() { + PrivateChild value = new PrivateChild(10, 20, -10, -20); + Fory fory = + Fory.builder() + .withXlang(false) + .withRefTracking(false) + .withCodegen(true) + .requireClassRegistration(false) + .build(); + Serializer serializer = + Serializers.newSerializer( + fory, + PrivateChild.class, + CodecUtils.loadOrGenObjectCodecClass(PrivateChild.class, fory)); + MemoryBuffer buffer = MemoryUtils.buffer(32); + writeSerializer(fory, serializer, buffer, value); + PrivateChild newValue = readSerializer(fory, serializer, buffer); + assertEquals(newValue.baseValue(), value.baseValue()); + assertEquals(newValue.baseFinalValue(), value.baseFinalValue()); + assertEquals(newValue.childValue(), value.childValue()); + assertEquals(newValue.childFinalValue(), value.childFinalValue()); + } + @Test public void testDuplicateFieldsCompatible() { C c = new C(); diff --git a/java/fory-core/src/test/java/org/apache/fory/serializer/FinalFieldReplaceResolveSerializerTest.java b/java/fory-core/src/test/java/org/apache/fory/serializer/FinalFieldReplaceResolveSerializerTest.java index a033c56018..49ca224047 100644 --- a/java/fory-core/src/test/java/org/apache/fory/serializer/FinalFieldReplaceResolveSerializerTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/serializer/FinalFieldReplaceResolveSerializerTest.java @@ -397,7 +397,6 @@ public void testNoClassNameWrittenForFinalField(boolean codegen) { byte[] bytesFinal = fory.serialize(containerFinal); byte[] bytesFinal2 = fory.serialize(containerFinal); assertEquals(bytesFinal, bytesFinal2); - assertEquals(bytesFinal.length, 109); // Create a container with a non-final ImmutableList field for comparison ContainerWithNonFinalImmutableIntArray containerNonFinal = @@ -405,7 +404,6 @@ public void testNoClassNameWrittenForFinalField(boolean codegen) { byte[] bytesNonFinal = fory.serialize(containerNonFinal); // The final field version should use fewer bytes because it doesn't write class name - System.out.println(bytesFinal.length + " " + bytesNonFinal.length); assertTrue( bytesFinal.length < bytesNonFinal.length, String.format( diff --git a/java/fory-core/src/test/java/org/apache/fory/serializer/ObjectSerializerTest.java b/java/fory-core/src/test/java/org/apache/fory/serializer/ObjectSerializerTest.java index 162b984c6f..8eb70dffa5 100644 --- a/java/fory-core/src/test/java/org/apache/fory/serializer/ObjectSerializerTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/serializer/ObjectSerializerTest.java @@ -21,7 +21,9 @@ import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNotSame; +import static org.testng.Assert.assertSame; +import java.beans.ConstructorProperties; import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; @@ -30,9 +32,13 @@ import lombok.Data; import org.apache.fory.Fory; import org.apache.fory.ForyTestBase; +import org.apache.fory.annotation.ForyField; +import org.apache.fory.builder.CodecUtils; import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.memory.MemoryUtils; +import org.apache.fory.meta.TypeDef; import org.apache.fory.platform.AndroidSupport; +import org.apache.fory.platform.JdkVersion; import org.apache.fory.test.bean.Cyclic; import org.apache.fory.util.Preconditions; import org.testng.Assert; @@ -171,6 +177,463 @@ public void testCopyCircularReference(Fory fory) { assertNotSame(cyclic1, cyclic); } + public static final class ConstructorCycle { + private final String name; + private ConstructorCycle next; + + @ConstructorProperties("name") + public ConstructorCycle(String name) { + this.name = name; + } + } + + public static final class ConstructorCycleBeforeFinal { + @ForyField(id = 0) + private ConstructorCycleBeforeFinal next; + + @ForyField(id = 1) + private final String name; + + @ConstructorProperties("name") + public ConstructorCycleBeforeFinal(String name) { + this.name = name; + } + } + + public static final class ConstructorOrder { + private int id; + private final String name; + + @ConstructorProperties("name") + public ConstructorOrder(String name) { + this.name = name; + } + } + + public static final class ConstructorInterveningRef { + @ForyField(id = 0) + private Object first; + + @ForyField(id = 1) + private final String name; + + @ForyField(id = 2) + private Object second; + + @ConstructorProperties("name") + public ConstructorInterveningRef(String name) { + this.name = name; + } + } + + public static final class ConstructorBackrefRoot { + private final ConstructorBackrefChild child; + + @ConstructorProperties("child") + public ConstructorBackrefRoot(ConstructorBackrefChild child) { + this.child = child; + } + } + + public static final class ConstructorBackrefChild { + private ConstructorBackrefRoot root; + } + + @Test + public void testConstructorFieldProtocolOrder() { + ConstructorOrder value = new ConstructorOrder("root"); + value.id = 42; + Fory fory = + Fory.builder() + .withXlang(false) + .withRefTracking(true) + .withCodegen(false) + .withNumberCompressed(false) + .requireClassRegistration(false) + .build(); + ObjectSerializer serializer = + new ObjectSerializer<>(fory.getTypeResolver(), ConstructorOrder.class); + MemoryBuffer buffer = MemoryUtils.buffer(32); + withWriteContext(fory, buffer, context -> serializer.write(context, value)); + assertEquals(buffer.readInt32(), 42); + } + + @Test + public void testConstructorFieldProtocolOrderCodegen() { + ConstructorOrder value = new ConstructorOrder("root"); + value.id = 42; + Fory fory = + Fory.builder() + .withXlang(false) + .withRefTracking(true) + .withCodegen(true) + .withNumberCompressed(false) + .requireClassRegistration(false) + .build(); + Serializer serializer = + Serializers.newSerializer( + fory, + ConstructorOrder.class, + CodecUtils.loadOrGenObjectCodecClass(ConstructorOrder.class, fory)); + MemoryBuffer buffer = MemoryUtils.buffer(32); + withWriteContext(fory, buffer, context -> serializer.write(context, value)); + assertEquals(buffer.readInt32(), 42); + } + + @Test + public void testCtorInterveningRef() { + Fory fory = + Fory.builder() + .withXlang(false) + .withRefTracking(true) + .withCodegen(false) + .requireClassRegistration(false) + .build(); + ConstructorInterveningRef newValue = + roundTripWithSerializer( + fory, + new ObjectSerializer<>(fory.getTypeResolver(), ConstructorInterveningRef.class), + newConstructorInterveningRef()); + assertInterveningRef(newValue); + } + + @Test + public void testCtorInterveningRefCodegen() { + Fory fory = + Fory.builder() + .withXlang(false) + .withRefTracking(true) + .withCodegen(true) + .requireClassRegistration(false) + .build(); + Serializer serializer = + Serializers.newSerializer( + fory, + ConstructorInterveningRef.class, + CodecUtils.loadOrGenObjectCodecClass(ConstructorInterveningRef.class, fory)); + ConstructorInterveningRef newValue = + roundTripWithSerializer(fory, serializer, newConstructorInterveningRef()); + assertInterveningRef(newValue); + } + + @Test + public void testCtorInterveningRefCompat() { + Fory fory = + Fory.builder() + .withXlang(false) + .withRefTracking(true) + .withCompatible(true) + .withCodegen(false) + .requireClassRegistration(false) + .build(); + TypeDef typeDef = fory.getTypeResolver().getTypeDef(ConstructorInterveningRef.class, true); + CompatibleSerializer serializer = + new CompatibleSerializer<>( + fory.getTypeResolver(), ConstructorInterveningRef.class, typeDef); + ConstructorInterveningRef newValue = + roundTripWithSerializer(fory, serializer, newConstructorInterveningRef()); + assertInterveningRef(newValue); + } + + @Test + public void testCtorInterveningRefCompatGen() { + Fory fory = + Fory.builder() + .withXlang(false) + .withRefTracking(true) + .withCompatible(true) + .withCodegen(true) + .requireClassRegistration(false) + .build(); + TypeDef typeDef = fory.getTypeResolver().getTypeDef(ConstructorInterveningRef.class, true); + Serializer serializer = + Serializers.newSerializer( + fory, + ConstructorInterveningRef.class, + CodecUtils.loadOrGenCompatibleCodecClass( + fory, ConstructorInterveningRef.class, typeDef)); + ConstructorInterveningRef newValue = + roundTripWithSerializer(fory, serializer, newConstructorInterveningRef()); + assertInterveningRef(newValue); + } + + @Test + public void testConstructorFieldBackrefRejected() { + if (JdkVersion.MAJOR_VERSION < 25) { + return; + } + ConstructorBackrefChild child = new ConstructorBackrefChild(); + ConstructorBackrefRoot value = new ConstructorBackrefRoot(child); + child.root = value; + Fory fory = + Fory.builder() + .withXlang(false) + .withRefTracking(true) + .withCodegen(false) + .requireClassRegistration(false) + .build(); + ObjectSerializer serializer = + new ObjectSerializer<>(fory.getTypeResolver(), ConstructorBackrefRoot.class); + MemoryBuffer buffer = MemoryUtils.buffer(32); + withWriteContext( + fory, + buffer, + context -> { + context.writeRefOrNull(value); + serializer.write(context, value); + }); + Assert.assertThrows( + org.apache.fory.exception.ForyException.class, + () -> + withReadContext( + fory, + buffer, + context -> { + byte tag = context.readRefOrNull(); + Preconditions.checkArgument(tag == Fory.REF_VALUE_FLAG); + context.preserveRefId(); + return serializer.read(context); + })); + } + + @Test + public void testConstructorFieldCycle() { + ConstructorCycle value = new ConstructorCycle("root"); + value.next = value; + Fory fory = + Fory.builder() + .withXlang(false) + .withRefTracking(true) + .withCodegen(false) + .requireClassRegistration(false) + .build(); + ConstructorCycle newValue = + roundTripWithSerializer( + fory, new ObjectSerializer<>(fory.getTypeResolver(), ConstructorCycle.class), value); + assertEquals(newValue.name, value.name); + assertSame(newValue.next, newValue); + } + + @Test + public void testConstructorFieldCycleBeforeFinal() { + ConstructorCycleBeforeFinal value = new ConstructorCycleBeforeFinal("root"); + value.next = value; + Fory fory = + Fory.builder() + .withXlang(false) + .withRefTracking(true) + .withCodegen(false) + .requireClassRegistration(false) + .build(); + ConstructorCycleBeforeFinal newValue = + roundTripWithSerializer( + fory, + new ObjectSerializer<>(fory.getTypeResolver(), ConstructorCycleBeforeFinal.class), + value); + assertEquals(newValue.name, value.name); + assertSame(newValue.next, newValue); + } + + @Test + public void testConstructorFieldCycleCodegen() { + ConstructorCycle value = new ConstructorCycle("root"); + value.next = value; + Fory fory = + Fory.builder() + .withXlang(false) + .withRefTracking(true) + .withCodegen(true) + .requireClassRegistration(false) + .build(); + Serializer serializer = + Serializers.newSerializer( + fory, + ConstructorCycle.class, + CodecUtils.loadOrGenObjectCodecClass(ConstructorCycle.class, fory)); + ConstructorCycle newValue = roundTripWithSerializer(fory, serializer, value); + assertEquals(newValue.name, value.name); + assertSame(newValue.next, newValue); + } + + @Test + public void testConstructorFieldCycleBeforeFinalCodegen() { + ConstructorCycleBeforeFinal value = new ConstructorCycleBeforeFinal("root"); + value.next = value; + Fory fory = + Fory.builder() + .withXlang(false) + .withRefTracking(true) + .withCodegen(true) + .requireClassRegistration(false) + .build(); + Serializer serializer = + Serializers.newSerializer( + fory, + ConstructorCycleBeforeFinal.class, + CodecUtils.loadOrGenObjectCodecClass(ConstructorCycleBeforeFinal.class, fory)); + ConstructorCycleBeforeFinal newValue = roundTripWithSerializer(fory, serializer, value); + assertEquals(newValue.name, value.name); + assertSame(newValue.next, newValue); + } + + @Test + public void testConstructorFieldCycleCompatible() { + ConstructorCycle value = new ConstructorCycle("root"); + value.next = value; + Fory fory = + Fory.builder() + .withXlang(false) + .withRefTracking(true) + .withCompatible(true) + .withCodegen(true) + .requireClassRegistration(false) + .build(); + TypeDef typeDef = fory.getTypeResolver().getTypeDef(ConstructorCycle.class, true); + Serializer serializer = + Serializers.newSerializer( + fory, + ConstructorCycle.class, + CodecUtils.loadOrGenCompatibleCodecClass(fory, ConstructorCycle.class, typeDef)); + ConstructorCycle newValue = roundTripWithSerializer(fory, serializer, value); + assertEquals(newValue.name, value.name); + assertSame(newValue.next, newValue); + } + + @Test + public void testConstructorFieldCycleBeforeFinalCompatible() { + ConstructorCycleBeforeFinal value = new ConstructorCycleBeforeFinal("root"); + value.next = value; + Fory fory = + Fory.builder() + .withXlang(false) + .withRefTracking(true) + .withCompatible(true) + .withCodegen(true) + .requireClassRegistration(false) + .build(); + TypeDef typeDef = fory.getTypeResolver().getTypeDef(ConstructorCycleBeforeFinal.class, true); + Serializer serializer = + Serializers.newSerializer( + fory, + ConstructorCycleBeforeFinal.class, + CodecUtils.loadOrGenCompatibleCodecClass( + fory, ConstructorCycleBeforeFinal.class, typeDef)); + ConstructorCycleBeforeFinal newValue = roundTripWithSerializer(fory, serializer, value); + assertEquals(newValue.name, value.name); + assertSame(newValue.next, newValue); + } + + @Test + public void testConstructorFieldCycleCompatibleNonCodegen() { + ConstructorCycleBeforeFinal value = new ConstructorCycleBeforeFinal("root"); + value.next = value; + Fory fory = + Fory.builder() + .withXlang(false) + .withRefTracking(true) + .withCompatible(true) + .withCodegen(false) + .requireClassRegistration(false) + .build(); + TypeDef typeDef = fory.getTypeResolver().getTypeDef(ConstructorCycleBeforeFinal.class, true); + CompatibleSerializer serializer = + new CompatibleSerializer<>( + fory.getTypeResolver(), ConstructorCycleBeforeFinal.class, typeDef); + ConstructorCycleBeforeFinal newValue = roundTripWithSerializer(fory, serializer, value); + assertEquals(newValue.name, value.name); + assertSame(newValue.next, newValue); + } + + @Test + public void testConstructorFieldBackrefCompatibleRejected() { + if (JdkVersion.MAJOR_VERSION < 25) { + return; + } + ConstructorBackrefChild child = new ConstructorBackrefChild(); + ConstructorBackrefRoot value = new ConstructorBackrefRoot(child); + child.root = value; + Fory fory = + Fory.builder() + .withXlang(false) + .withRefTracking(true) + .withCompatible(true) + .withCodegen(false) + .requireClassRegistration(false) + .build(); + TypeDef typeDef = fory.getTypeResolver().getTypeDef(ConstructorBackrefRoot.class, true); + CompatibleSerializer serializer = + new CompatibleSerializer<>(fory.getTypeResolver(), ConstructorBackrefRoot.class, typeDef); + MemoryBuffer buffer = MemoryUtils.buffer(32); + withWriteContext( + fory, + buffer, + context -> { + context.writeRefOrNull(value); + serializer.write(context, value); + }); + Assert.assertThrows( + org.apache.fory.exception.ForyException.class, + () -> + withReadContext( + fory, + buffer, + context -> { + byte tag = context.readRefOrNull(); + Preconditions.checkArgument(tag == Fory.REF_VALUE_FLAG); + context.preserveRefId(); + return serializer.read(context); + })); + } + + @Test(dataProvider = "foryCopyConfig") + public void testConstructorFieldCycleCopy(Fory fory) { + ConstructorCycle value = new ConstructorCycle("root"); + value.next = value; + ObjectSerializer serializer = + new ObjectSerializer<>(fory.getTypeResolver(), ConstructorCycle.class); + ConstructorCycle newValue = withCopyContext(fory, context -> serializer.copy(context, value)); + assertEquals(newValue.name, value.name); + assertSame(newValue.next, newValue); + } + + private static T roundTripWithSerializer(Fory fory, Serializer serializer, T value) { + MemoryBuffer buffer = MemoryUtils.buffer(32); + withWriteContext( + fory, + buffer, + context -> { + context.writeRefOrNull(value); + serializer.write(context, value); + }); + T newValue = + withReadContext( + fory, + buffer, + context -> { + byte tag = context.readRefOrNull(); + Preconditions.checkArgument(tag == Fory.REF_VALUE_FLAG); + context.preserveRefId(); + return serializer.read(context); + }); + fory.reset(); + return newValue; + } + + private static ConstructorInterveningRef newConstructorInterveningRef() { + ConstructorInterveningRef value = new ConstructorInterveningRef("root"); + Object shared = new String("shared"); + value.first = shared; + value.second = shared; + return value; + } + + private static void assertInterveningRef(ConstructorInterveningRef value) { + assertEquals(value.name, "root"); + assertEquals(value.first, "shared"); + assertSame(value.second, value.first); + Assert.assertNotSame(value.second, value); + } + @Data public static class A { Integer f1; diff --git a/java/fory-core/src/test/java/org/apache/fory/serializer/ObjectStreamSerializerTest.java b/java/fory-core/src/test/java/org/apache/fory/serializer/ObjectStreamSerializerTest.java index d623d609c7..df49ec926d 100644 --- a/java/fory-core/src/test/java/org/apache/fory/serializer/ObjectStreamSerializerTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/serializer/ObjectStreamSerializerTest.java @@ -494,7 +494,7 @@ public void testWriteObjectReplace(Fory fory) throws MalformedURLException { new ObjectStreamSerializer(fory.getTypeResolver(), WriteObjectTestClass4.class)); Assert.assertEquals( - serDeCheckSerializer(fory, new URL("http://test"), "ReplaceResolve"), + serDeCheckSerializer(fory, new URL("http://test"), "URLSerializer"), new URL("http://test")); WriteObjectTestClass4 testClassObj4 = new WriteObjectTestClass4(new char[] {'a', 'b'}); serDeCheckSerializer(fory, testClassObj4, "ObjectStreamSerializer"); diff --git a/java/fory-core/src/test/java/org/apache/fory/serializer/ReplaceResolveSerializerTest.java b/java/fory-core/src/test/java/org/apache/fory/serializer/ReplaceResolveSerializerTest.java index 027f006174..67f7cac9d9 100644 --- a/java/fory-core/src/test/java/org/apache/fory/serializer/ReplaceResolveSerializerTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/serializer/ReplaceResolveSerializerTest.java @@ -621,6 +621,8 @@ static class ReplaceSelfExternalizable implements Externalizable { private transient int f1; private transient boolean newInstance; + public ReplaceSelfExternalizable() {} + public ReplaceSelfExternalizable(int f1, boolean newInstance) { this.f1 = f1; this.newInstance = newInstance; diff --git a/java/fory-core/src/test/java/org/apache/fory/serializer/URLSerializerTest.java b/java/fory-core/src/test/java/org/apache/fory/serializer/URLSerializerTest.java index 5035051921..5da5aa16c6 100644 --- a/java/fory-core/src/test/java/org/apache/fory/serializer/URLSerializerTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/serializer/URLSerializerTest.java @@ -31,7 +31,7 @@ public class URLSerializerTest extends ForyTestBase { @Test(dataProvider = "javaFory") public void testDefaultWrite(Fory fory) throws MalformedURLException { Assert.assertEquals( - serDeCheckSerializer(fory, new URL("http://test"), "ReplaceResolve"), + serDeCheckSerializer(fory, new URL("http://test"), "URLSerializer"), new URL("http://test")); } diff --git a/java/fory-core/src/test/java/org/apache/fory/serializer/collection/CollectionSerializersTest.java b/java/fory-core/src/test/java/org/apache/fory/serializer/collection/CollectionSerializersTest.java index 3f9e4b1d72..7eb639c0d1 100644 --- a/java/fory-core/src/test/java/org/apache/fory/serializer/collection/CollectionSerializersTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/serializer/collection/CollectionSerializersTest.java @@ -28,6 +28,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import com.google.common.collect.ImmutableSortedSet; +import java.beans.ConstructorProperties; import java.io.Externalizable; import java.io.IOException; import java.io.ObjectInput; @@ -881,10 +882,15 @@ public void testCopyOnWriteArrayList(Fory fory) { } @Data - @AllArgsConstructor public static class CollectionViewTestStruct { Collection collection; Set set; + + @ConstructorProperties({"collection", "set"}) + public CollectionViewTestStruct(Collection collection, Set set) { + this.collection = collection; + this.set = set; + } } @Test(dataProvider = "javaFory") @@ -1369,7 +1375,16 @@ public void testNestedCollection2Copy(Fory fory) { } public static class TestClassForDefaultCollectionSerializer extends AbstractCollection { - private final List data = new ArrayList<>(); + private final List data; + + public TestClassForDefaultCollectionSerializer() { + this(new ArrayList<>()); + } + + @ConstructorProperties({"data"}) + public TestClassForDefaultCollectionSerializer(List data) { + this.data = data; + } @Override public Iterator iterator() { diff --git a/java/fory-core/src/test/java/org/apache/fory/serializer/collection/MapSerializersTest.java b/java/fory-core/src/test/java/org/apache/fory/serializer/collection/MapSerializersTest.java index 2db9d5c842..ef69240218 100644 --- a/java/fory-core/src/test/java/org/apache/fory/serializer/collection/MapSerializersTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/serializer/collection/MapSerializersTest.java @@ -26,6 +26,7 @@ import static org.testng.Assert.assertEquals; import com.google.common.collect.ImmutableMap; +import java.beans.ConstructorProperties; import java.io.Externalizable; import java.io.IOException; import java.io.ObjectInput; @@ -825,7 +826,16 @@ public static MapFields createMapFieldsObject(Map map) { } public static class TestClass1ForDefaultMap extends AbstractMap { - private final Set data = new HashSet<>(); + private final Set data; + + public TestClass1ForDefaultMap() { + this(new HashSet<>()); + } + + @ConstructorProperties({"data"}) + public TestClass1ForDefaultMap(Set data) { + this.data = data; + } @Override public Set> entrySet() { @@ -840,7 +850,16 @@ public Object put(String key, Object value) { } public static class TestClass2ForDefaultMap extends AbstractMap { - private final Set> data = new HashSet<>(); + private final Set> data; + + public TestClass2ForDefaultMap() { + this(new HashSet<>()); + } + + @ConstructorProperties({"data"}) + public TestClass2ForDefaultMap(Set> data) { + this.data = data; + } @Override public Set> entrySet() { @@ -1177,10 +1196,16 @@ public int hashCode() { } @Data - @AllArgsConstructor public static class LazyMapCollectionFieldStruct { List> mapList; PrivateMap map; + + @ConstructorProperties({"mapList", "map"}) + LazyMapCollectionFieldStruct( + List> mapList, PrivateMap map) { + this.mapList = mapList; + this.map = map; + } } @Data @@ -1387,8 +1412,19 @@ public void testNestedMapFieldStructCodegen(boolean referenceTrackingConfig) { @Data public static class PrivateFinalMapFieldStruct { - private final Map valueMap = new LinkedHashMap<>(); - private final Map keyMap = new LinkedHashMap<>(); + private final Map valueMap; + private final Map keyMap; + + public PrivateFinalMapFieldStruct() { + this(new LinkedHashMap<>(), new LinkedHashMap<>()); + } + + @ConstructorProperties({"valueMap", "keyMap"}) + public PrivateFinalMapFieldStruct( + Map valueMap, Map keyMap) { + this.valueMap = valueMap; + this.keyMap = keyMap; + } } @Data diff --git a/java/fory-format/pom.xml b/java/fory-format/pom.xml index 01cdd773ec..15deca64b3 100644 --- a/java/fory-format/pom.xml +++ b/java/fory-format/pom.xml @@ -105,6 +105,7 @@ + true org.apache.fory.format @@ -114,7 +115,10 @@ org.apache.maven.plugins maven-surefire-plugin - --add-opens=java.base/java.nio=ALL-UNNAMED + + --add-opens=java.base/java.nio=ALL-UNNAMED + --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED + @@ -177,6 +181,15 @@ + + + + true + org.apache.fory.format + + + + diff --git a/java/fory-format/src/main/java/org/apache/fory/format/vectorized/ArrowUtils.java b/java/fory-format/src/main/java/org/apache/fory/format/vectorized/ArrowUtils.java index 0d93894bf5..226117b7df 100644 --- a/java/fory-format/src/main/java/org/apache/fory/format/vectorized/ArrowUtils.java +++ b/java/fory-format/src/main/java/org/apache/fory/format/vectorized/ArrowUtils.java @@ -37,15 +37,44 @@ /** Arrow utils. */ public class ArrowUtils { + private static final RuntimeException ALLOCATOR_ERROR; + // RootAllocator is thread-safe, so we don't have to use thread-local. - // FIXME JDK17: Unable to make field long java.nio.Buffer.address - // accessible: module java.base does not "opens java.nio" to unnamed module @405e4200 - public static RootAllocator allocator = new RootAllocator(); + // Arrow 18.x initializes its own sun.misc.Unsafe memory facade eagerly. Keep Fory class loading + // possible under JDK25 deny mode and fail only when the vectorized Arrow path is used. + public static RootAllocator allocator; + + static { + RootAllocator rootAllocator = null; + RuntimeException allocatorError = null; + if (isUnsafeMemoryDenied()) { + allocatorError = + new UnsupportedOperationException( + "Apache Arrow vectorized format is unavailable when JDK Unsafe memory access is " + + "denied. Apache Arrow initializes sun.misc.Unsafe memory access internally."); + } else { + try { + rootAllocator = new RootAllocator(); + } catch (RuntimeException | ExceptionInInitializerError e) { + if (!isUnsafeMemoryAccessFailure(e)) { + throw e; + } + allocatorError = + new UnsupportedOperationException( + "Apache Arrow vectorized format is unavailable when Apache Arrow cannot initialize " + + "its sun.misc.Unsafe memory access.", + e); + } + } + allocator = rootAllocator; + ALLOCATOR_ERROR = allocatorError; + } + private static final ThreadLocal decimalArrowBuf = ThreadLocal.withInitial(() -> buffer(DecimalUtils.DECIMAL_BYTE_LENGTH)); public static ArrowBuf buffer(final long initialRequestSize) { - return allocator.buffer(initialRequestSize); + return requireAllocator().buffer(initialRequestSize); } public static ArrowBuf decimalArrowBuf() { @@ -53,22 +82,23 @@ public static ArrowBuf decimalArrowBuf() { } public static VectorSchemaRoot createVectorSchemaRoot(Schema arrowSchema) { - return VectorSchemaRoot.create(arrowSchema, allocator); + return VectorSchemaRoot.create(arrowSchema, requireAllocator()); } public static VectorSchemaRoot createVectorSchemaRoot( org.apache.fory.format.type.Schema forySchema) { - return VectorSchemaRoot.create(ArrowSchemaConverter.toArrowSchema(forySchema), allocator); + return VectorSchemaRoot.create( + ArrowSchemaConverter.toArrowSchema(forySchema), requireAllocator()); } public static ArrowWriter createArrowWriter(Schema arrowSchema) { - VectorSchemaRoot root = VectorSchemaRoot.create(arrowSchema, allocator); + VectorSchemaRoot root = VectorSchemaRoot.create(arrowSchema, requireAllocator()); return new ArrowWriter(root); } public static ArrowWriter createArrowWriter(org.apache.fory.format.type.Schema forySchema) { VectorSchemaRoot root = - VectorSchemaRoot.create(ArrowSchemaConverter.toArrowSchema(forySchema), allocator); + VectorSchemaRoot.create(ArrowSchemaConverter.toArrowSchema(forySchema), requireAllocator()); return new ArrowWriter(root); } @@ -87,9 +117,41 @@ public static ArrowRecordBatch deserializeRecordBatch(MemoryBuffer recordBatchMe try (ReadChannel channel = new ReadChannel( Channels.newChannel(new MemoryBufferInputStream(recordBatchMessageBuffer)))) { - return MessageSerializer.deserializeRecordBatch(channel, allocator); + return MessageSerializer.deserializeRecordBatch(channel, requireAllocator()); } catch (IOException e) { throw new RuntimeException("Deserialize record batch failed", e); } } + + private static RootAllocator requireAllocator() { + if (allocator != null) { + return allocator; + } + throw ALLOCATOR_ERROR; + } + + private static boolean isUnsafeMemoryDenied() { + return Runtime.version().feature() >= 25 + && "deny".equals(System.getProperty("sun.misc.unsafe.memory.access")); + } + + private static boolean isUnsafeMemoryAccessFailure(Throwable throwable) { + Throwable cause = throwable; + while (cause != null) { + if (cause instanceof UnsupportedOperationException + && cause.getMessage() != null + && cause.getMessage().contains("arrayBaseOffset")) { + return true; + } + if (cause instanceof UnsupportedOperationException) { + for (StackTraceElement element : cause.getStackTrace()) { + if ("sun.misc.Unsafe".equals(element.getClassName())) { + return true; + } + } + } + cause = cause.getCause(); + } + return false; + } } diff --git a/java/fory-format/src/test/java/org/apache/fory/format/vectorized/ArrowTestSupport.java b/java/fory-format/src/test/java/org/apache/fory/format/vectorized/ArrowTestSupport.java new file mode 100644 index 0000000000..ff46da5bca --- /dev/null +++ b/java/fory-format/src/test/java/org/apache/fory/format/vectorized/ArrowTestSupport.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.format.vectorized; + +import org.apache.arrow.memory.ArrowBuf; +import org.testng.SkipException; + +final class ArrowTestSupport { + private ArrowTestSupport() {} + + static void skipIfArrowUnavailable() { + try (ArrowBuf ignored = ArrowUtils.buffer(0)) { + // No-op. This verifies that Arrow's allocator can initialize in this JVM mode. + } catch (UnsupportedOperationException e) { + throw new SkipException(e.getMessage(), e); + } + } +} diff --git a/java/fory-format/src/test/java/org/apache/fory/format/vectorized/ArrowUtilsTest.java b/java/fory-format/src/test/java/org/apache/fory/format/vectorized/ArrowUtilsTest.java index 48a00cc5aa..4b8079734c 100644 --- a/java/fory-format/src/test/java/org/apache/fory/format/vectorized/ArrowUtilsTest.java +++ b/java/fory-format/src/test/java/org/apache/fory/format/vectorized/ArrowUtilsTest.java @@ -53,6 +53,7 @@ public static VectorSchemaRoot createVectorSchemaRoot(int size) { @Test public void testSerializeRecordBatch() { + ArrowTestSupport.skipIfArrowUnavailable(); VectorSchemaRoot vectorSchemaRoot = createVectorSchemaRoot(2); VectorUnloader unloader = new VectorUnloader(vectorSchemaRoot); ArrowRecordBatch recordBatch = unloader.getRecordBatch(); diff --git a/java/fory-format/src/test/java/org/apache/fory/format/vectorized/ArrowWriterTest.java b/java/fory-format/src/test/java/org/apache/fory/format/vectorized/ArrowWriterTest.java index 66df4ff5d2..ffc83f2fde 100644 --- a/java/fory-format/src/test/java/org/apache/fory/format/vectorized/ArrowWriterTest.java +++ b/java/fory-format/src/test/java/org/apache/fory/format/vectorized/ArrowWriterTest.java @@ -60,6 +60,7 @@ private ArrowRecordBatch createArrowRecordBatch() { @Test public void testWrite() { + ArrowTestSupport.skipIfArrowUnavailable(); ArrowRecordBatch recordBatch = createArrowRecordBatch(); System.out.println("recordBatch " + recordBatch); recordBatch.close(); @@ -67,6 +68,7 @@ public void testWrite() { @Test public void testSerializeArrowRecordBatch() { + ArrowTestSupport.skipIfArrowUnavailable(); ArrowRecordBatch recordBatch = createArrowRecordBatch(); System.out.println("recordBatch serialized body size " + recordBatch.computeBodyLength()); MemoryBuffer buffer = MemoryUtils.buffer(32); diff --git a/java/fory-latest-jdk-tests/src/test/java/org/apache/fory/integration_tests/ImmutableCollectionSerializersTest.java b/java/fory-latest-jdk-tests/src/test/java/org/apache/fory/integration_tests/ImmutableCollectionSerializersTest.java index ae7db28118..0b7983a120 100644 --- a/java/fory-latest-jdk-tests/src/test/java/org/apache/fory/integration_tests/ImmutableCollectionSerializersTest.java +++ b/java/fory-latest-jdk-tests/src/test/java/org/apache/fory/integration_tests/ImmutableCollectionSerializersTest.java @@ -21,10 +21,10 @@ import static org.apache.fory.integration_tests.TestUtils.serDeCheck; +import java.beans.ConstructorProperties; import java.util.List; import java.util.Map; import java.util.Set; -import lombok.AllArgsConstructor; import lombok.Data; import org.apache.fory.Fory; import org.apache.fory.ThreadSafeFory; @@ -94,9 +94,13 @@ public void testImmutableMapStruct() { } @Data - @AllArgsConstructor public static class Pojo { List> data; + + @ConstructorProperties("data") + public Pojo(List> data) { + this.data = data; + } } @DataProvider diff --git a/java/fory-test-core/src/main/java/org/apache/fory/test/bean/AccessBeans.java b/java/fory-test-core/src/main/java/org/apache/fory/test/bean/AccessBeans.java index 132d5def40..7b7dfab896 100644 --- a/java/fory-test-core/src/main/java/org/apache/fory/test/bean/AccessBeans.java +++ b/java/fory-test-core/src/main/java/org/apache/fory/test/bean/AccessBeans.java @@ -19,36 +19,53 @@ package org.apache.fory.test.bean; -import lombok.AllArgsConstructor; +import java.beans.ConstructorProperties; import lombok.Data; public class AccessBeans { @Data - @AllArgsConstructor private static class PrivateClass { public int f1; int f2; private int f3; + + @ConstructorProperties({"f1", "f2", "f3"}) + PrivateClass(int f1, int f2, int f3) { + this.f1 = f1; + this.f2 = f2; + this.f3 = f3; + } } @Data - @AllArgsConstructor private static final class FinalPrivateClass { public int f1; int f2; private int f3; + + @ConstructorProperties({"f1", "f2", "f3"}) + FinalPrivateClass(int f1, int f2, int f3) { + this.f1 = f1; + this.f2 = f2; + this.f3 = f3; + } } @Data - @AllArgsConstructor static class DefaultLevelClass { public int f1; int f2; private int f3; + + @ConstructorProperties({"f1", "f2", "f3"}) + DefaultLevelClass(int f1, int f2, int f3) { + this.f1 = f1; + this.f2 = f2; + this.f3 = f3; + } } @Data - @AllArgsConstructor public static class PublicClass { public int f1; int f2; @@ -56,6 +73,17 @@ public static class PublicClass { private DefaultLevelClass f4; private PrivateClass f5; private FinalPrivateClass f6; + + @ConstructorProperties({"f1", "f2", "f3", "f4", "f5", "f6"}) + public PublicClass( + int f1, int f2, int f3, DefaultLevelClass f4, PrivateClass f5, FinalPrivateClass f6) { + this.f1 = f1; + this.f2 = f2; + this.f3 = f3; + this.f4 = f4; + this.f5 = f5; + this.f6 = f6; + } } public static PrivateClass createPrivateClassObject() { diff --git a/java/fory-testsuite/src/test/java/org/apache/fory/serializer/collection/GuavaCollectionSerializersTest.java b/java/fory-testsuite/src/test/java/org/apache/fory/serializer/collection/GuavaCollectionSerializersTest.java index b103a1467f..744ec9d0a7 100644 --- a/java/fory-testsuite/src/test/java/org/apache/fory/serializer/collection/GuavaCollectionSerializersTest.java +++ b/java/fory-testsuite/src/test/java/org/apache/fory/serializer/collection/GuavaCollectionSerializersTest.java @@ -25,6 +25,7 @@ import com.google.common.collect.ImmutableSet; import com.google.common.collect.ImmutableSortedMap; import com.google.common.collect.ImmutableSortedSet; +import java.beans.ConstructorProperties; import java.util.List; import java.util.Objects; import org.apache.fory.Fory; @@ -229,6 +230,7 @@ public void testNestedRefTrackingCopy(Fory fory) { public static final class Pojo { private final List> data; + @ConstructorProperties("data") public Pojo(List> data) { this.data = data; } diff --git a/java/fory-testsuite/src/test/java/org/apache/fory/test/FastJsonTest.java b/java/fory-testsuite/src/test/java/org/apache/fory/test/FastJsonTest.java index 841d0b0028..08cdecf248 100644 --- a/java/fory-testsuite/src/test/java/org/apache/fory/test/FastJsonTest.java +++ b/java/fory-testsuite/src/test/java/org/apache/fory/test/FastJsonTest.java @@ -22,6 +22,7 @@ import com.alibaba.fastjson.JSONObject; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Sets; +import java.beans.ConstructorProperties; import java.util.List; import org.apache.fory.Fory; import org.apache.fory.collection.Collections; @@ -34,6 +35,7 @@ public static class DemoResponse { private JSONObject json; private List objects; + @ConstructorProperties("json") public DemoResponse(JSONObject json) { this.json = json; objects = Collections.ofArrayList(json); diff --git a/java/fory-testsuite/src/test/java/org/apache/fory/test/ReadResolveCircularTest.java b/java/fory-testsuite/src/test/java/org/apache/fory/test/ReadResolveCircularTest.java index 37133b1be6..47d397869c 100644 --- a/java/fory-testsuite/src/test/java/org/apache/fory/test/ReadResolveCircularTest.java +++ b/java/fory-testsuite/src/test/java/org/apache/fory/test/ReadResolveCircularTest.java @@ -95,9 +95,16 @@ private static final class SerializationProxy implements Serializable { private String containerLabel = ""; private List items; + private SerializationProxy() {} + public SerializationProxy(Container c) { this.containerLabel = c.getLabel(); - this.items = new ArrayList<>(c.getItems()); + this.items = new ArrayList<>(); + for (Item item : c.getItems()) { + // The proxy reconstructs parent links in readResolve; serializing the original back-link + // would point Item.parent at the proxy object after writeReplace. + this.items.add(new Item(item.getName())); + } } private Object readResolve() { diff --git a/java/pom.xml b/java/pom.xml index 4e1d14ae43..c51ebe940d 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -109,6 +109,38 @@ fory-latest-jdk-tests + + + jdk25-test-classpath + + [25,] + + fory.jdk25.test.classpath + true + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + + org.apache.fory:fory-core + + + + ${maven.multiModuleProjectDirectory}/fory-core/target/jdk25-test-classes + + + + + + + From b8ce0ab69b7e8f8fa3d567feb6da05426de2bf7b Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sun, 24 May 2026 14:18:44 +0800 Subject: [PATCH 15/69] chore(java): fix JDK25 memory buffer license header --- .../java25/org/apache/fory/memory/MemoryBuffer.java | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/java/fory-core/src/main/java25/org/apache/fory/memory/MemoryBuffer.java b/java/fory-core/src/main/java25/org/apache/fory/memory/MemoryBuffer.java index 82203215cf..30c5d7b0e5 100644 --- a/java/fory-core/src/main/java25/org/apache/fory/memory/MemoryBuffer.java +++ b/java/fory-core/src/main/java25/org/apache/fory/memory/MemoryBuffer.java @@ -7,13 +7,14 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package org.apache.fory.memory; From 287d5ef25aecbd16581658e2315a0b64285e7d31 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sun, 24 May 2026 14:21:35 +0800 Subject: [PATCH 16/69] fix(java): keep string coder lookup Java 8 safe --- .../java/org/apache/fory/platform/internal/_JDKAccess.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/java/fory-core/src/main/java/org/apache/fory/platform/internal/_JDKAccess.java b/java/fory-core/src/main/java/org/apache/fory/platform/internal/_JDKAccess.java index e4c9d47ad9..88e8b2e9ae 100644 --- a/java/fory-core/src/main/java/org/apache/fory/platform/internal/_JDKAccess.java +++ b/java/fory-core/src/main/java/org/apache/fory/platform/internal/_JDKAccess.java @@ -167,14 +167,15 @@ private static class StringCoderField { } } - public static final long STRING_CODER_FIELD_OFFSET = StringCoderField.OFFSET; + public static final long STRING_CODER_FIELD_OFFSET = + STRING_VALUE_FIELD_IS_BYTES ? StringCoderField.OFFSET : -1; public static Object getStringValue(String value) { return UNSAFE.getObject(value, STRING_VALUE_FIELD_OFFSET); } public static byte getStringCoder(String value) { - return UNSAFE.getByte(value, StringCoderField.OFFSET); + return UNSAFE.getByte(value, STRING_CODER_FIELD_OFFSET); } public static int getStringOffset(String value) { From e947b5cdb8b3d4d845f9395b83038340d1539c0a Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sun, 24 May 2026 14:27:38 +0800 Subject: [PATCH 17/69] fix(ci): repair style and kotlin xlang peer jar --- .../java/org/apache/fory/context/MapRefReader.java | 12 ++++++------ kotlin/fory-kotlin-tests/pom.xml | 11 +++++++++++ 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/java/fory-core/src/main/java/org/apache/fory/context/MapRefReader.java b/java/fory-core/src/main/java/org/apache/fory/context/MapRefReader.java index 6613349580..328acbf78d 100644 --- a/java/fory-core/src/main/java/org/apache/fory/context/MapRefReader.java +++ b/java/fory-core/src/main/java/org/apache/fory/context/MapRefReader.java @@ -132,6 +132,12 @@ public Object getReadRef(int id) { return readObjects.get(id); } + /** Returns the object resolved by the last ref header that pointed to an existing instance. */ + @Override + public Object getReadRef() { + return readObject; + } + private Object readRef(int id) { if (trackedRefIds.size == 0) { return readObjects.get(id); @@ -152,12 +158,6 @@ private boolean isTrackedRef(int id) { return false; } - /** Returns the object resolved by the last ref header that pointed to an existing instance. */ - @Override - public Object getReadRef() { - return readObject; - } - /** Stores {@code object} under an already reserved read ref id. */ @Override public void setReadRef(int id, Object object) { diff --git a/kotlin/fory-kotlin-tests/pom.xml b/kotlin/fory-kotlin-tests/pom.xml index f8e0409b8f..2aad72b680 100644 --- a/kotlin/fory-kotlin-tests/pom.xml +++ b/kotlin/fory-kotlin-tests/pom.xml @@ -105,6 +105,7 @@ compile + true ${project.basedir}/src/main/kotlin ${project.build.directory}/generated-sources/ksp @@ -125,6 +126,16 @@ fory-kotlin-tests-xlang-peer false + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + From e9303f8529d281a6c1ca0620208a081afcd352ce Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sun, 24 May 2026 14:43:08 +0800 Subject: [PATCH 18/69] fix(ci): avoid android and graal field access failures --- .../org/apache/fory/collection/MapEntry.java | 2 -- .../org/apache/fory/memory/MemoryUtils.java | 29 ++++++++++++++----- .../java/org/apache/fory/type/BFloat16.java | 2 -- .../org/apache/fory/type/BFloat16Array.java | 2 -- .../java/org/apache/fory/type/Float16.java | 2 -- .../org/apache/fory/type/Float16Array.java | 2 -- .../org/apache/fory/type/unsigned/UInt16.java | 2 -- .../org/apache/fory/type/unsigned/UInt32.java | 2 -- .../org/apache/fory/type/unsigned/UInt64.java | 2 -- .../org/apache/fory/type/unsigned/UInt8.java | 2 -- java/pom.xml | 1 + 11 files changed, 23 insertions(+), 25 deletions(-) diff --git a/java/fory-core/src/main/java/org/apache/fory/collection/MapEntry.java b/java/fory-core/src/main/java/org/apache/fory/collection/MapEntry.java index 5ec07a7974..b0bc8c3337 100644 --- a/java/fory-core/src/main/java/org/apache/fory/collection/MapEntry.java +++ b/java/fory-core/src/main/java/org/apache/fory/collection/MapEntry.java @@ -19,7 +19,6 @@ package org.apache.fory.collection; -import java.beans.ConstructorProperties; import java.util.Map; import java.util.Objects; @@ -28,7 +27,6 @@ public class MapEntry implements Map.Entry { private final K key; private V value; - @ConstructorProperties({"key", "value"}) public MapEntry(K key, V value) { this.key = key; this.value = value; diff --git a/java/fory-core/src/main/java/org/apache/fory/memory/MemoryUtils.java b/java/fory-core/src/main/java/org/apache/fory/memory/MemoryUtils.java index 81d35bb4d1..41db2cfe46 100644 --- a/java/fory-core/src/main/java/org/apache/fory/memory/MemoryUtils.java +++ b/java/fory-core/src/main/java/org/apache/fory/memory/MemoryUtils.java @@ -23,6 +23,7 @@ import java.io.ByteArrayOutputStream; import java.nio.ByteBuffer; import org.apache.fory.platform.AndroidSupport; +import org.apache.fory.platform.GraalvmSupport; import org.apache.fory.platform.internal._JDKAccess; /** Memory utils for fory. */ @@ -31,19 +32,33 @@ public class MemoryUtils { // When a JDK25+ path needs JDK private fields, open the needed java.base package to both // org.apache.fory.core and org.apache.fory.format. public static final boolean JDK_INTERNAL_FIELD_ACCESS = - !AndroidSupport.IS_ANDROID && _JDKAccess.JDK_INTERNAL_FIELD_ACCESS; + !AndroidSupport.IS_ANDROID + && !GraalvmSupport.IN_GRAALVM_NATIVE_IMAGE + && _JDKAccess.JDK_INTERNAL_FIELD_ACCESS; public static final boolean JDK_LANG_FIELD_ACCESS = - !AndroidSupport.IS_ANDROID && _JDKAccess.JDK_LANG_FIELD_ACCESS; + !AndroidSupport.IS_ANDROID + && !GraalvmSupport.IN_GRAALVM_NATIVE_IMAGE + && _JDKAccess.JDK_LANG_FIELD_ACCESS; public static final boolean JDK_BYTE_ARRAY_STREAM_FIELD_ACCESS = - !AndroidSupport.IS_ANDROID && _JDKAccess.JDK_BYTE_ARRAY_STREAM_FIELD_ACCESS; + !AndroidSupport.IS_ANDROID + && !GraalvmSupport.IN_GRAALVM_NATIVE_IMAGE + && _JDKAccess.JDK_BYTE_ARRAY_STREAM_FIELD_ACCESS; public static final boolean JDK_OBJECT_STREAM_FIELD_ACCESS = - !AndroidSupport.IS_ANDROID && _JDKAccess.JDK_OBJECT_STREAM_FIELD_ACCESS; + !AndroidSupport.IS_ANDROID + && !GraalvmSupport.IN_GRAALVM_NATIVE_IMAGE + && _JDKAccess.JDK_OBJECT_STREAM_FIELD_ACCESS; public static final boolean JDK_COLLECTION_FIELD_ACCESS = - !AndroidSupport.IS_ANDROID && _JDKAccess.JDK_COLLECTION_FIELD_ACCESS; + !AndroidSupport.IS_ANDROID + && !GraalvmSupport.IN_GRAALVM_NATIVE_IMAGE + && _JDKAccess.JDK_COLLECTION_FIELD_ACCESS; public static final boolean JDK_CONCURRENT_FIELD_ACCESS = - !AndroidSupport.IS_ANDROID && _JDKAccess.JDK_CONCURRENT_FIELD_ACCESS; + !AndroidSupport.IS_ANDROID + && !GraalvmSupport.IN_GRAALVM_NATIVE_IMAGE + && _JDKAccess.JDK_CONCURRENT_FIELD_ACCESS; public static final boolean JDK_PROXY_FIELD_ACCESS = - !AndroidSupport.IS_ANDROID && _JDKAccess.JDK_PROXY_FIELD_ACCESS; + !AndroidSupport.IS_ANDROID + && !GraalvmSupport.IN_GRAALVM_NATIVE_IMAGE + && _JDKAccess.JDK_PROXY_FIELD_ACCESS; public static MemoryBuffer buffer(int size) { return wrap(new byte[size]); diff --git a/java/fory-core/src/main/java/org/apache/fory/type/BFloat16.java b/java/fory-core/src/main/java/org/apache/fory/type/BFloat16.java index b1e0f2b67e..fb0d905ffb 100644 --- a/java/fory-core/src/main/java/org/apache/fory/type/BFloat16.java +++ b/java/fory-core/src/main/java/org/apache/fory/type/BFloat16.java @@ -19,7 +19,6 @@ package org.apache.fory.type; -import java.beans.ConstructorProperties; import java.io.Serializable; /** @@ -70,7 +69,6 @@ public final class BFloat16 extends Number implements Comparable, Seri private final short bits; - @ConstructorProperties("bits") private BFloat16(short bits) { this.bits = bits; } diff --git a/java/fory-core/src/main/java/org/apache/fory/type/BFloat16Array.java b/java/fory-core/src/main/java/org/apache/fory/type/BFloat16Array.java index 538b7095bb..21eae6959a 100644 --- a/java/fory-core/src/main/java/org/apache/fory/type/BFloat16Array.java +++ b/java/fory-core/src/main/java/org/apache/fory/type/BFloat16Array.java @@ -19,7 +19,6 @@ package org.apache.fory.type; -import java.beans.ConstructorProperties; import java.io.Serializable; import java.util.Arrays; import java.util.Iterator; @@ -46,7 +45,6 @@ public BFloat16Array(BFloat16[] values) { } } - @ConstructorProperties("bits") private BFloat16Array(short[] bits) { this.bits = bits; } diff --git a/java/fory-core/src/main/java/org/apache/fory/type/Float16.java b/java/fory-core/src/main/java/org/apache/fory/type/Float16.java index 562df4d180..c76eaa362c 100644 --- a/java/fory-core/src/main/java/org/apache/fory/type/Float16.java +++ b/java/fory-core/src/main/java/org/apache/fory/type/Float16.java @@ -19,7 +19,6 @@ package org.apache.fory.type; -import java.beans.ConstructorProperties; import java.io.Serializable; public final class Float16 extends Number implements Comparable, Serializable { @@ -62,7 +61,6 @@ public final class Float16 extends Number implements Comparable, Serial private final short bits; - @ConstructorProperties("bits") private Float16(short bits) { this.bits = bits; } diff --git a/java/fory-core/src/main/java/org/apache/fory/type/Float16Array.java b/java/fory-core/src/main/java/org/apache/fory/type/Float16Array.java index f94e618f72..e724a2dbae 100644 --- a/java/fory-core/src/main/java/org/apache/fory/type/Float16Array.java +++ b/java/fory-core/src/main/java/org/apache/fory/type/Float16Array.java @@ -19,7 +19,6 @@ package org.apache.fory.type; -import java.beans.ConstructorProperties; import java.io.Serializable; import java.util.Arrays; import java.util.Iterator; @@ -46,7 +45,6 @@ public Float16Array(Float16[] values) { } } - @ConstructorProperties("bits") private Float16Array(short[] bits) { this.bits = bits; } diff --git a/java/fory-core/src/main/java/org/apache/fory/type/unsigned/UInt16.java b/java/fory-core/src/main/java/org/apache/fory/type/unsigned/UInt16.java index 320260816c..cd57b1dab1 100644 --- a/java/fory-core/src/main/java/org/apache/fory/type/unsigned/UInt16.java +++ b/java/fory-core/src/main/java/org/apache/fory/type/unsigned/UInt16.java @@ -19,7 +19,6 @@ package org.apache.fory.type.unsigned; -import java.beans.ConstructorProperties; import java.io.Serializable; /** @@ -40,7 +39,6 @@ public final class UInt16 implements Comparable, Serializable { private final short data; - @ConstructorProperties("data") public UInt16(short data) { this.data = data; } diff --git a/java/fory-core/src/main/java/org/apache/fory/type/unsigned/UInt32.java b/java/fory-core/src/main/java/org/apache/fory/type/unsigned/UInt32.java index 3f8052e69a..2a1a4999fd 100644 --- a/java/fory-core/src/main/java/org/apache/fory/type/unsigned/UInt32.java +++ b/java/fory-core/src/main/java/org/apache/fory/type/unsigned/UInt32.java @@ -19,7 +19,6 @@ package org.apache.fory.type.unsigned; -import java.beans.ConstructorProperties; import java.io.Serializable; /** @@ -38,7 +37,6 @@ public final class UInt32 implements Comparable, Serializable { private final int data; - @ConstructorProperties("data") public UInt32(int data) { this.data = data; } diff --git a/java/fory-core/src/main/java/org/apache/fory/type/unsigned/UInt64.java b/java/fory-core/src/main/java/org/apache/fory/type/unsigned/UInt64.java index 3caff91f83..d5c7267c9f 100644 --- a/java/fory-core/src/main/java/org/apache/fory/type/unsigned/UInt64.java +++ b/java/fory-core/src/main/java/org/apache/fory/type/unsigned/UInt64.java @@ -19,7 +19,6 @@ package org.apache.fory.type.unsigned; -import java.beans.ConstructorProperties; import java.io.Serializable; /** @@ -37,7 +36,6 @@ public final class UInt64 implements Comparable, Serializable { private final long data; - @ConstructorProperties("data") public UInt64(long data) { this.data = data; } diff --git a/java/fory-core/src/main/java/org/apache/fory/type/unsigned/UInt8.java b/java/fory-core/src/main/java/org/apache/fory/type/unsigned/UInt8.java index 70eb002787..b4183108f1 100644 --- a/java/fory-core/src/main/java/org/apache/fory/type/unsigned/UInt8.java +++ b/java/fory-core/src/main/java/org/apache/fory/type/unsigned/UInt8.java @@ -19,7 +19,6 @@ package org.apache.fory.type.unsigned; -import java.beans.ConstructorProperties; import java.io.Serializable; /** @@ -40,7 +39,6 @@ public final class UInt8 implements Comparable, Serializable { private final byte data; - @ConstructorProperties("data") public UInt8(byte data) { this.data = data; } diff --git a/java/pom.xml b/java/pom.xml index c51ebe940d..7884abb1bc 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -270,6 +270,7 @@ ${maven.compiler.source} ${maven.compiler.target} + true From c8b8ddacfce8db9da0af1158e2a8bb9f8c2f68c4 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sun, 24 May 2026 14:48:27 +0800 Subject: [PATCH 19/69] style(ci): format java task helper --- ci/tasks/java.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/ci/tasks/java.py b/ci/tasks/java.py index cf8bb10041..bed7db53fc 100644 --- a/ci/tasks/java.py +++ b/ci/tasks/java.py @@ -153,9 +153,7 @@ def install_jdk25_fory_artifacts(): ) logging.info("Verify JDK25 benchmark multi-release jar") common.cd_project_subdir("benchmarks/java") - common.exec_cmd( - "mvn -T10 -B --no-transfer-progress -Pjmh -DskipTests install" - ) + common.exec_cmd("mvn -T10 -B --no-transfer-progress -Pjmh -DskipTests install") logging.info("Verify JPMS tests on JDK25") common.cd_project_subdir("integration_tests/jpms_tests") common.exec_cmd("mvn -T10 -B --no-transfer-progress clean test") From 999fd9214d41f899c3e70ff3ad6560a41b46f4ed Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sun, 24 May 2026 14:56:01 +0800 Subject: [PATCH 20/69] fix(android): skip jdk string internals on android --- .../fory/platform/internal/_JDKAccess.java | 94 ++++++++++++------- 1 file changed, 58 insertions(+), 36 deletions(-) diff --git a/java/fory-core/src/main/java/org/apache/fory/platform/internal/_JDKAccess.java b/java/fory-core/src/main/java/org/apache/fory/platform/internal/_JDKAccess.java index 88e8b2e9ae..7019213ed0 100644 --- a/java/fory-core/src/main/java/org/apache/fory/platform/internal/_JDKAccess.java +++ b/java/fory-core/src/main/java/org/apache/fory/platform/internal/_JDKAccess.java @@ -45,6 +45,7 @@ import org.apache.fory.collection.ClassValueCache; import org.apache.fory.collection.Tuple2; import org.apache.fory.memory.MemoryBuffer; +import org.apache.fory.platform.AndroidSupport; import org.apache.fory.platform.GraalvmSupport; import org.apache.fory.platform.JdkVersion; import org.apache.fory.type.TypeUtils; @@ -88,27 +89,39 @@ public class _JDKAccess { throw new UnsupportedOperationException("Unsafe is not supported in this platform."); } UNSAFE = unsafe; - JDK_INTERNAL_FIELD_ACCESS = true; - JDK_LANG_FIELD_ACCESS = true; - JDK_STRING_FIELD_ACCESS = true; - JDK_BYTE_ARRAY_STREAM_FIELD_ACCESS = true; - JDK_OBJECT_STREAM_FIELD_ACCESS = true; - JDK_COLLECTION_FIELD_ACCESS = true; - JDK_CONCURRENT_FIELD_ACCESS = true; - JDK_PROXY_FIELD_ACCESS = true; - if (JdkVersion.MAJOR_VERSION >= 11) { + if (AndroidSupport.IS_ANDROID) { + JDK_INTERNAL_FIELD_ACCESS = false; + JDK_LANG_FIELD_ACCESS = false; + JDK_STRING_FIELD_ACCESS = false; + JDK_BYTE_ARRAY_STREAM_FIELD_ACCESS = false; + JDK_OBJECT_STREAM_FIELD_ACCESS = false; + JDK_COLLECTION_FIELD_ACCESS = false; + JDK_CONCURRENT_FIELD_ACCESS = false; + JDK_PROXY_FIELD_ACCESS = false; + } else { + JDK_INTERNAL_FIELD_ACCESS = true; + JDK_LANG_FIELD_ACCESS = true; + JDK_STRING_FIELD_ACCESS = true; + JDK_BYTE_ARRAY_STREAM_FIELD_ACCESS = true; + JDK_OBJECT_STREAM_FIELD_ACCESS = true; + JDK_COLLECTION_FIELD_ACCESS = true; + JDK_CONCURRENT_FIELD_ACCESS = true; + JDK_PROXY_FIELD_ACCESS = true; + } + Object innerUnsafe = null; + Class innerUnsafeClass = null; + if (!AndroidSupport.IS_ANDROID && JdkVersion.MAJOR_VERSION >= 11) { try { Field theInternalUnsafeField = Unsafe.class.getDeclaredField("theInternalUnsafe"); theInternalUnsafeField.setAccessible(true); - _INNER_UNSAFE = theInternalUnsafeField.get(null); - _INNER_UNSAFE_CLASS = _INNER_UNSAFE.getClass(); + innerUnsafe = theInternalUnsafeField.get(null); + innerUnsafeClass = innerUnsafe.getClass(); } catch (Exception e) { throw new RuntimeException(e); } - } else { - _INNER_UNSAFE_CLASS = null; - _INNER_UNSAFE = null; } + _INNER_UNSAFE = innerUnsafe; + _INNER_UNSAFE_CLASS = innerUnsafeClass; } private static final ClassValueCache lookupCache = ClassValueCache.newClassKeyCache(32); @@ -121,29 +134,38 @@ public class _JDKAccess { public static final long STRING_OFFSET_FIELD_OFFSET; static { - try { - Field valueField = String.class.getDeclaredField("value"); - STRING_VALUE_FIELD_IS_CHARS = valueField.getType() == char[].class; - STRING_VALUE_FIELD_IS_BYTES = valueField.getType() == byte[].class; - STRING_VALUE_FIELD_OFFSET = UNSAFE.objectFieldOffset(valueField); - Field countField = getStringFieldNullable("count"); - Field offsetField = getStringFieldNullable("offset"); - if (countField != null || offsetField != null) { - Preconditions.checkArgument( - countField != null && offsetField != null, "Current jdk not supported"); - Preconditions.checkArgument( - countField.getType() == int.class && offsetField.getType() == int.class, - "Current jdk not supported"); - STRING_HAS_COUNT_OFFSET = true; - STRING_COUNT_FIELD_OFFSET = UNSAFE.objectFieldOffset(countField); - STRING_OFFSET_FIELD_OFFSET = UNSAFE.objectFieldOffset(offsetField); - } else { - STRING_HAS_COUNT_OFFSET = false; - STRING_COUNT_FIELD_OFFSET = -1; - STRING_OFFSET_FIELD_OFFSET = -1; + if (AndroidSupport.IS_ANDROID) { + STRING_VALUE_FIELD_IS_CHARS = false; + STRING_VALUE_FIELD_IS_BYTES = false; + STRING_VALUE_FIELD_OFFSET = -1; + STRING_HAS_COUNT_OFFSET = false; + STRING_COUNT_FIELD_OFFSET = -1; + STRING_OFFSET_FIELD_OFFSET = -1; + } else { + try { + Field valueField = String.class.getDeclaredField("value"); + STRING_VALUE_FIELD_IS_CHARS = valueField.getType() == char[].class; + STRING_VALUE_FIELD_IS_BYTES = valueField.getType() == byte[].class; + STRING_VALUE_FIELD_OFFSET = UNSAFE.objectFieldOffset(valueField); + Field countField = getStringFieldNullable("count"); + Field offsetField = getStringFieldNullable("offset"); + if (countField != null || offsetField != null) { + Preconditions.checkArgument( + countField != null && offsetField != null, "Current jdk not supported"); + Preconditions.checkArgument( + countField.getType() == int.class && offsetField.getType() == int.class, + "Current jdk not supported"); + STRING_HAS_COUNT_OFFSET = true; + STRING_COUNT_FIELD_OFFSET = UNSAFE.objectFieldOffset(countField); + STRING_OFFSET_FIELD_OFFSET = UNSAFE.objectFieldOffset(offsetField); + } else { + STRING_HAS_COUNT_OFFSET = false; + STRING_COUNT_FIELD_OFFSET = -1; + STRING_OFFSET_FIELD_OFFSET = -1; + } + } catch (NoSuchFieldException e) { + throw new RuntimeException(e); } - } catch (NoSuchFieldException e) { - throw new RuntimeException(e); } } From e56a62d27aba447b3c294292bc80a2116222c308 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sun, 24 May 2026 15:07:22 +0800 Subject: [PATCH 21/69] fix(graalvm): use public string access in native images --- .../java/org/apache/fory/serializer/StringSerializer.java | 7 ++++++- .../org/apache/fory/serializer/StringSerializer.java | 7 ++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/StringSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/StringSerializer.java index 8e9fa43965..078bc109bc 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/StringSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/StringSerializer.java @@ -36,6 +36,7 @@ import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.memory.NativeByteOrder; import org.apache.fory.platform.AndroidSupport; +import org.apache.fory.platform.GraalvmSupport; import org.apache.fory.platform.UnsafeOps; import org.apache.fory.platform.internal._JDKAccess; import org.apache.fory.util.MathUtils; @@ -84,7 +85,11 @@ public final class StringSerializer extends ImmutableSerializer { } private static boolean jdkInternalFieldAccess() { - return !AndroidSupport.IS_ANDROID && _JDKAccess.JDK_STRING_FIELD_ACCESS; + // Native-image runtime string layout is not a HotSpot Unsafe-offset contract; use public + // string copies there even when the image build can see JDK private fields. + return !AndroidSupport.IS_ANDROID + && !GraalvmSupport.IN_GRAALVM_NATIVE_IMAGE + && _JDKAccess.JDK_STRING_FIELD_ACCESS; } private final boolean compressString; diff --git a/java/fory-core/src/main/java25/org/apache/fory/serializer/StringSerializer.java b/java/fory-core/src/main/java25/org/apache/fory/serializer/StringSerializer.java index e290be4069..92cbb62154 100644 --- a/java/fory-core/src/main/java25/org/apache/fory/serializer/StringSerializer.java +++ b/java/fory-core/src/main/java25/org/apache/fory/serializer/StringSerializer.java @@ -36,6 +36,7 @@ import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.memory.NativeByteOrder; import org.apache.fory.platform.AndroidSupport; +import org.apache.fory.platform.GraalvmSupport; import org.apache.fory.platform.UnsafeOps; import org.apache.fory.platform.internal._JDKAccess; import org.apache.fory.util.MathUtils; @@ -75,7 +76,11 @@ public final class StringSerializer extends ImmutableSerializer { } private static boolean jdkInternalFieldAccess() { - return !AndroidSupport.IS_ANDROID && _JDKAccess.JDK_STRING_FIELD_ACCESS; + // Native-image runtime string layout is not a HotSpot private-field contract; use public + // string copies there even when the image build can see JDK private fields. + return !AndroidSupport.IS_ANDROID + && !GraalvmSupport.IN_GRAALVM_NATIVE_IMAGE + && _JDKAccess.JDK_STRING_FIELD_ACCESS; } private final boolean compressString; From 607a180ff75bf03ae742d19fdcdb13586b58a960 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sun, 24 May 2026 15:11:44 +0800 Subject: [PATCH 22/69] fix(graalvm): avoid private string codegen in native images --- .../java/org/apache/fory/serializer/StringSerializer.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/StringSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/StringSerializer.java index 078bc109bc..8dafbccdd9 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/StringSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/StringSerializer.java @@ -127,6 +127,9 @@ public String read(ReadContext readContext) { public static Expression writeStringExpr( Expression strSerializer, Expression buffer, Expression str, boolean compressString) { + if (!jdkInternalFieldAccess()) { + return new Invoke(strSerializer, "writeString", buffer, str); + } if (STRING_VALUE_FIELD_IS_BYTES) { if (compressString) { return new Invoke(strSerializer, "writeCompressedBytesString", buffer, str); @@ -155,6 +158,9 @@ public static Expression writeStringExpr( public static Expression readStringExpr( Expression strSerializer, Expression buffer, boolean compressString) { + if (!jdkInternalFieldAccess()) { + return new Invoke(strSerializer, "readString", STRING_TYPE, buffer); + } if (STRING_VALUE_FIELD_IS_BYTES) { if (compressString) { return new Invoke(strSerializer, "readCompressedBytesString", STRING_TYPE, buffer); From 047f712cf5925c91740b6ce7fc7d333b662d1eb0 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sun, 24 May 2026 16:16:01 +0800 Subject: [PATCH 23/69] perf(java): add JDK25 direct memory benchmark --- benchmarks/java25/README.md | 32 +++ benchmarks/java25/pom.xml | 115 ++++++++++ .../java25/DirectMemoryAccessBenchmark.java | 216 ++++++++++++++++++ 3 files changed, 363 insertions(+) create mode 100644 benchmarks/java25/README.md create mode 100644 benchmarks/java25/pom.xml create mode 100644 benchmarks/java25/src/main/java/org/apache/fory/benchmark/java25/DirectMemoryAccessBenchmark.java diff --git a/benchmarks/java25/README.md b/benchmarks/java25/README.md new file mode 100644 index 0000000000..c50663989d --- /dev/null +++ b/benchmarks/java25/README.md @@ -0,0 +1,32 @@ +# Java 25 Direct Memory Access Benchmark + +This temporary JMH module compares direct-buffer scalar access paths used to reason about +`MemoryBuffer` on JDK 25: + +- `MemorySegment.get/set` with native-order unaligned layouts. +- `MethodHandles.byteBufferViewVarHandle` over a direct `ByteBuffer`. +- `sun.misc.Unsafe` raw native-address access over the same direct `ByteBuffer`. + +Each benchmark invocation performs one absolute scalar access at a rolling aligned offset. This is +closer to generated serializer calls into `MemoryBuffer.writeInt32`, `_unsafePutInt64`, and matching +read paths than a bulk array-copy benchmark. + +Build and run with JDK 25: + +```bash +cd benchmarks/java25 +mvn package +java -jar target/java25-memory-access-benchmarks.jar \ + 'org.apache.fory.benchmark.java25.DirectMemoryAccessBenchmark.*' \ + -f 1 -wi 5 -i 5 -t 1 -w 1s -r 1s +``` + +The benchmark class adds the required fork JVM options for the Unsafe path: + +```text +--add-opens=java.base/java.nio=ALL-UNNAMED +--sun-misc-unsafe-memory-access=allow +``` + +If you run with `-f 0`, pass those options to the outer `java` command because JMH will not fork a +child JVM. diff --git a/benchmarks/java25/pom.xml b/benchmarks/java25/pom.xml new file mode 100644 index 0000000000..72dd1d1ae3 --- /dev/null +++ b/benchmarks/java25/pom.xml @@ -0,0 +1,115 @@ + + + + 4.0.0 + + org.apache.fory.benchmark + java25-memory-access-benchmark + 1.1.0-SNAPSHOT + jar + + + UTF-8 + 1.33 + 25 + true + true + java25-memory-access-benchmarks + + + + + org.openjdk.jmh + jmh-core + ${jmh.version} + + + org.openjdk.jmh + jmh-generator-annprocess + ${jmh.version} + provided + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.14.0 + + ${maven.compiler.release} + + + org.openjdk.jmh + jmh-generator-annprocess + ${jmh.version} + + + + + + org.apache.maven.plugins + maven-shade-plugin + 3.6.0 + + + package + + shade + + + ${uberjar.name} + false + + + org.openjdk.jmh.Main + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + + + + org.apache.maven.plugins + maven-deploy-plugin + 3.1.4 + + true + + + + + diff --git a/benchmarks/java25/src/main/java/org/apache/fory/benchmark/java25/DirectMemoryAccessBenchmark.java b/benchmarks/java25/src/main/java/org/apache/fory/benchmark/java25/DirectMemoryAccessBenchmark.java new file mode 100644 index 0000000000..da251382cf --- /dev/null +++ b/benchmarks/java25/src/main/java/org/apache/fory/benchmark/java25/DirectMemoryAccessBenchmark.java @@ -0,0 +1,216 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.benchmark.java25; + +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; +import java.lang.invoke.MethodHandles; +import java.lang.invoke.VarHandle; +import java.lang.reflect.Field; +import java.nio.Buffer; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.concurrent.TimeUnit; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Threads; +import org.openjdk.jmh.annotations.Warmup; +import sun.misc.Unsafe; + +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.NANOSECONDS) +@Warmup(iterations = 5, time = 1) +@Measurement(iterations = 5, time = 1) +@Fork( + value = 1, + jvmArgsAppend = { + "--add-opens=java.base/java.nio=ALL-UNNAMED", + "--sun-misc-unsafe-memory-access=allow" + }) +@Threads(1) +public class DirectMemoryAccessBenchmark { + private static final int BUFFER_BYTES = 64 * 1024; + private static final int INT_SLOTS = BUFFER_BYTES / Integer.BYTES; + private static final int LONG_SLOTS = BUFFER_BYTES / Long.BYTES; + private static final int INT_SLOT_MASK = INT_SLOTS - 1; + private static final int LONG_SLOT_MASK = LONG_SLOTS - 1; + private static final ByteOrder NATIVE_ORDER = ByteOrder.nativeOrder(); + private static final Unsafe UNSAFE = loadUnsafe(); + private static final long BUFFER_ADDRESS_OFFSET = bufferAddressOffset(); + private static final VarHandle INT_HANDLE = + MethodHandles.byteBufferViewVarHandle(int[].class, NATIVE_ORDER); + private static final VarHandle LONG_HANDLE = + MethodHandles.byteBufferViewVarHandle(long[].class, NATIVE_ORDER); + private static final ValueLayout.OfInt INT_LAYOUT = + ValueLayout.JAVA_INT_UNALIGNED.withOrder(NATIVE_ORDER); + private static final ValueLayout.OfLong LONG_LAYOUT = + ValueLayout.JAVA_LONG_UNALIGNED.withOrder(NATIVE_ORDER); + + @State(Scope.Thread) + public static class DirectState { + ByteBuffer buffer; + MemorySegment segment; + long address; + int intValue; + int intCursor; + long longValue; + int longCursor; + + @Setup + public void setup() { + buffer = ByteBuffer.allocateDirect(BUFFER_BYTES).order(NATIVE_ORDER); + segment = MemorySegment.ofBuffer(buffer); + address = UNSAFE.getLong(buffer, BUFFER_ADDRESS_OFFSET); + intValue = 0x12345678; + longValue = 0x123456789abcdef0L; + for (int i = 0; i < INT_SLOTS; i++) { + int intOffset = i << 2; + int intPattern = intValue + i; + UNSAFE.putInt(null, address + intOffset, intPattern); + } + for (int i = 0; i < LONG_SLOTS; i++) { + int longOffset = i << 3; + long longPattern = longValue + i; + UNSAFE.putLong(null, address + longOffset, longPattern); + } + } + + int nextIntOffset() { + return (intCursor++ & INT_SLOT_MASK) << 2; + } + + int nextIntValue() { + intValue += 0x9e3779b9; + return intValue; + } + + int nextLongOffset() { + return (longCursor++ & LONG_SLOT_MASK) << 3; + } + + long nextLongValue() { + longValue += 0x9e3779b97f4a7c15L; + return longValue; + } + } + + @Benchmark + public int memorySegmentPutInt(DirectState state) { + int offset = state.nextIntOffset(); + int value = state.nextIntValue(); + state.segment.set(INT_LAYOUT, offset, value); + return value; + } + + @Benchmark + public int varHandlePutInt(DirectState state) { + int offset = state.nextIntOffset(); + int value = state.nextIntValue(); + INT_HANDLE.set(state.buffer, offset, value); + return value; + } + + @Benchmark + public int unsafePutInt(DirectState state) { + int offset = state.nextIntOffset(); + int value = state.nextIntValue(); + UNSAFE.putInt(null, state.address + offset, value); + return value; + } + + @Benchmark + public int memorySegmentGetInt(DirectState state) { + return state.segment.get(INT_LAYOUT, state.nextIntOffset()); + } + + @Benchmark + public int varHandleGetInt(DirectState state) { + return (int) INT_HANDLE.get(state.buffer, state.nextIntOffset()); + } + + @Benchmark + public int unsafeGetInt(DirectState state) { + return UNSAFE.getInt(null, state.address + state.nextIntOffset()); + } + + @Benchmark + public long memorySegmentPutLong(DirectState state) { + int offset = state.nextLongOffset(); + long value = state.nextLongValue(); + state.segment.set(LONG_LAYOUT, offset, value); + return value; + } + + @Benchmark + public long varHandlePutLong(DirectState state) { + int offset = state.nextLongOffset(); + long value = state.nextLongValue(); + LONG_HANDLE.set(state.buffer, offset, value); + return value; + } + + @Benchmark + public long unsafePutLong(DirectState state) { + int offset = state.nextLongOffset(); + long value = state.nextLongValue(); + UNSAFE.putLong(null, state.address + offset, value); + return value; + } + + @Benchmark + public long memorySegmentGetLong(DirectState state) { + return state.segment.get(LONG_LAYOUT, state.nextLongOffset()); + } + + @Benchmark + public long varHandleGetLong(DirectState state) { + return (long) LONG_HANDLE.get(state.buffer, state.nextLongOffset()); + } + + @Benchmark + public long unsafeGetLong(DirectState state) { + return UNSAFE.getLong(null, state.address + state.nextLongOffset()); + } + + private static Unsafe loadUnsafe() { + try { + Field field = Unsafe.class.getDeclaredField("theUnsafe"); + field.setAccessible(true); + return (Unsafe) field.get(null); + } catch (ReflectiveOperationException e) { + throw new ExceptionInInitializerError(e); + } + } + + private static long bufferAddressOffset() { + try { + return UNSAFE.objectFieldOffset(Buffer.class.getDeclaredField("address")); + } catch (NoSuchFieldException e) { + throw new ExceptionInInitializerError(e); + } + } +} From 139431b5a9c61bd00f0e64cdc9dae38e3298c56a Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sun, 24 May 2026 16:29:49 +0800 Subject: [PATCH 24/69] perf(java): add JDK25 direct copy benchmark --- benchmarks/java25/README.md | 8 ++ .../java25/DirectToHeapCopyBenchmark.java | 123 ++++++++++++++++++ 2 files changed, 131 insertions(+) create mode 100644 benchmarks/java25/src/main/java/org/apache/fory/benchmark/java25/DirectToHeapCopyBenchmark.java diff --git a/benchmarks/java25/README.md b/benchmarks/java25/README.md index c50663989d..a7edad7c00 100644 --- a/benchmarks/java25/README.md +++ b/benchmarks/java25/README.md @@ -21,6 +21,14 @@ java -jar target/java25-memory-access-benchmarks.jar \ -f 1 -wi 5 -i 5 -t 1 -w 1s -r 1s ``` +Run the direct-to-heap copy benchmark: + +```bash +java -jar target/java25-memory-access-benchmarks.jar \ + 'org.apache.fory.benchmark.java25.DirectToHeapCopyBenchmark.*' \ + -f 1 -wi 5 -i 5 -t 1 -w 1s -r 1s +``` + The benchmark class adds the required fork JVM options for the Unsafe path: ```text diff --git a/benchmarks/java25/src/main/java/org/apache/fory/benchmark/java25/DirectToHeapCopyBenchmark.java b/benchmarks/java25/src/main/java/org/apache/fory/benchmark/java25/DirectToHeapCopyBenchmark.java new file mode 100644 index 0000000000..f3c3658471 --- /dev/null +++ b/benchmarks/java25/src/main/java/org/apache/fory/benchmark/java25/DirectToHeapCopyBenchmark.java @@ -0,0 +1,123 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.benchmark.java25; + +import java.lang.foreign.MemorySegment; +import java.lang.reflect.Field; +import java.nio.Buffer; +import java.nio.ByteBuffer; +import java.util.concurrent.TimeUnit; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Threads; +import org.openjdk.jmh.annotations.Warmup; +import sun.misc.Unsafe; + +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.NANOSECONDS) +@Warmup(iterations = 5, time = 1) +@Measurement(iterations = 5, time = 1) +@Fork( + value = 1, + jvmArgsAppend = { + "--add-opens=java.base/java.nio=ALL-UNNAMED", + "--sun-misc-unsafe-memory-access=allow" + }) +@Threads(1) +public class DirectToHeapCopyBenchmark { + private static final int BUFFER_BYTES = 64 * 1024; + private static final Unsafe UNSAFE = loadUnsafe(); + private static final long BUFFER_ADDRESS_OFFSET = bufferAddressOffset(); + private static final int BYTE_ARRAY_OFFSET = UNSAFE.arrayBaseOffset(byte[].class); + + @State(Scope.Thread) + public static class CopyState { + @Param({"128", "256", "512", "1024", "2048"}) + int copySize; + + ByteBuffer directBuffer; + MemorySegment directSegment; + byte[] heapBuffer; + MemorySegment heapSegment; + long directAddress; + + @Setup + public void setup() { + directBuffer = ByteBuffer.allocateDirect(BUFFER_BYTES); + directSegment = MemorySegment.ofBuffer(directBuffer); + heapBuffer = new byte[BUFFER_BYTES]; + heapSegment = MemorySegment.ofArray(heapBuffer); + directAddress = UNSAFE.getLong(directBuffer, BUFFER_ADDRESS_OFFSET); + for (int i = 0; i < BUFFER_BYTES; i++) { + UNSAFE.putByte(null, directAddress + i, (byte) (i * 31)); + } + } + } + + @Benchmark + public int byteBufferGet(CopyState state) { + int copySize = state.copySize; + byte[] heap = state.heapBuffer; + state.directBuffer.get(0, heap, 0, copySize); + return heap[copySize - 1]; + } + + @Benchmark + public int memorySegmentCopy(CopyState state) { + int copySize = state.copySize; + byte[] heap = state.heapBuffer; + MemorySegment.copy(state.directSegment, 0, state.heapSegment, 0, copySize); + return heap[copySize - 1]; + } + + @Benchmark + public int unsafeCopyMemory(CopyState state) { + int copySize = state.copySize; + byte[] heap = state.heapBuffer; + UNSAFE.copyMemory(null, state.directAddress, heap, BYTE_ARRAY_OFFSET, copySize); + return heap[copySize - 1]; + } + + private static Unsafe loadUnsafe() { + try { + Field field = Unsafe.class.getDeclaredField("theUnsafe"); + field.setAccessible(true); + return (Unsafe) field.get(null); + } catch (ReflectiveOperationException e) { + throw new ExceptionInInitializerError(e); + } + } + + private static long bufferAddressOffset() { + try { + return UNSAFE.objectFieldOffset(Buffer.class.getDeclaredField("address")); + } catch (NoSuchFieldException e) { + throw new ExceptionInInitializerError(e); + } + } +} From 79e7513a61e1540b4365aef33e26e7b95d681eaf Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sun, 24 May 2026 17:16:22 +0800 Subject: [PATCH 25/69] refactor(java): isolate string unsafe operations --- .../fory/benchmark/Jdk25MrJarCheck.java | 11 +- java/fory-core/pom.xml | 18 +- .../apache/fory/meta/MetaStringEncoder.java | 6 +- .../fory/serializer/PlatformStringUtils.java | 126 ++ .../apache/fory/serializer/Serializers.java | 3 +- .../fory/serializer/SlicedStringUtil.java | 296 ----- .../StringEncodingUtils.java | 318 ++++- .../fory/serializer/StringSerializer.java | 144 +- .../org/apache/fory/util/StringUtils.java | 35 - .../fory/serializer/PlatformStringUtils.java | 139 ++ .../apache/fory/serializer/Serializers.java | 3 +- .../fory/serializer/SlicedStringUtil.java | 285 ---- .../fory/serializer/StringSerializer.java | 1171 ----------------- .../StringEncodingUtilsTest.java | 2 +- .../org/apache/fory/util/StringUtilsTest.java | 35 +- 15 files changed, 630 insertions(+), 1962 deletions(-) create mode 100644 java/fory-core/src/main/java/org/apache/fory/serializer/PlatformStringUtils.java delete mode 100644 java/fory-core/src/main/java/org/apache/fory/serializer/SlicedStringUtil.java rename java/fory-core/src/main/java/org/apache/fory/{util => serializer}/StringEncodingUtils.java (59%) create mode 100644 java/fory-core/src/main/java25/org/apache/fory/serializer/PlatformStringUtils.java delete mode 100644 java/fory-core/src/main/java25/org/apache/fory/serializer/SlicedStringUtil.java delete mode 100644 java/fory-core/src/main/java25/org/apache/fory/serializer/StringSerializer.java rename java/fory-core/src/test/java/org/apache/fory/{util => serializer}/StringEncodingUtilsTest.java (98%) diff --git a/benchmarks/java/src/main/java/org/apache/fory/benchmark/Jdk25MrJarCheck.java b/benchmarks/java/src/main/java/org/apache/fory/benchmark/Jdk25MrJarCheck.java index 857b668e04..1bb2b38d91 100644 --- a/benchmarks/java/src/main/java/org/apache/fory/benchmark/Jdk25MrJarCheck.java +++ b/benchmarks/java/src/main/java/org/apache/fory/benchmark/Jdk25MrJarCheck.java @@ -23,7 +23,6 @@ import org.apache.fory.platform.UnsafeOps; import org.apache.fory.platform.internal._JDKAccess; import org.apache.fory.reflect.FieldAccessor; -import org.apache.fory.serializer.StringSerializer; /** Runtime smoke check that JDK25 benchmark runs load the multi-release Fory classes. */ public final class Jdk25MrJarCheck { @@ -34,12 +33,20 @@ public static void main(String[] args) { verifyClass(UnsafeOps.class); verifyClass(_JDKAccess.class); verifyClass(FieldAccessor.class); - verifyClass(StringSerializer.class); + verifyClass("org.apache.fory.serializer.PlatformStringUtils"); if (_JDKAccess.UNSAFE != null) { throw new IllegalStateException("JDK25 benchmark jar loaded Unsafe-backed _JDKAccess"); } } + private static void verifyClass(String className) { + try { + verifyClass(Class.forName(className)); + } catch (ClassNotFoundException e) { + throw new IllegalStateException("JDK25 benchmark jar is missing " + className, e); + } + } + private static void verifyClass(Class cls) { String resourceName = cls.getSimpleName() + ".class"; String resource = String.valueOf(cls.getResource(resourceName)); diff --git a/java/fory-core/pom.xml b/java/fory-core/pom.xml index f5a9a699ea..99ccc7a058 100644 --- a/java/fory-core/pom.xml +++ b/java/fory-core/pom.xml @@ -174,6 +174,7 @@ + @@ -251,8 +252,7 @@ name="META-INF/versions/25/org/apache/fory/reflect/HiddenFieldAccessorFactory.class"/> - - + - + file="${jdk25.mrjar.check.dir}/META-INF/versions/25/org/apache/fory/serializer/PlatformStringUtils.class" + property="jdk25.platformstring.present"/> @@ -328,11 +325,8 @@ unless="jdk25.serializers.present" message="JDK25 multi-release Serializers class is missing from the packaged fory-core jar."/> - + unless="jdk25.platformstring.present" + message="JDK25 multi-release PlatformStringUtils class is missing from the packaged fory-core jar."/> diff --git a/java/fory-core/src/main/java/org/apache/fory/meta/MetaStringEncoder.java b/java/fory-core/src/main/java/org/apache/fory/meta/MetaStringEncoder.java index e35460d8c5..ce4c976d71 100644 --- a/java/fory-core/src/main/java/org/apache/fory/meta/MetaStringEncoder.java +++ b/java/fory-core/src/main/java/org/apache/fory/meta/MetaStringEncoder.java @@ -21,8 +21,8 @@ import java.nio.charset.StandardCharsets; import org.apache.fory.meta.MetaString.Encoding; +import org.apache.fory.serializer.StringEncodingUtils; import org.apache.fory.util.Preconditions; -import org.apache.fory.util.StringUtils; /** Encodes plain text strings into MetaString objects with specified encoding mechanisms. */ public class MetaStringEncoder { @@ -78,7 +78,7 @@ public EncodedMetaString encodeBinary(String input, Encoding[] encodings) { if (input.isEmpty()) { return EncodedMetaString.EMPTY; } - if (!StringUtils.isLatin(input.toCharArray())) { + if (!StringEncodingUtils.isLatin(input.toCharArray())) { return new EncodedMetaString(input.getBytes(StandardCharsets.UTF_8), Encoding.UTF_8); } Encoding encoding = computeEncoding(input, encodings); @@ -88,7 +88,7 @@ public EncodedMetaString encodeBinary(String input, Encoding[] encodings) { public EncodedMetaString encodeBinary(String input, Encoding encoding) { Preconditions.checkArgument( input.length() < Short.MAX_VALUE, "Long meta string than 32767 is not allowed"); - if (encoding != Encoding.UTF_8 && !StringUtils.isLatin(input.toCharArray())) { + if (encoding != Encoding.UTF_8 && !StringEncodingUtils.isLatin(input.toCharArray())) { throw new IllegalArgumentException("Non-ASCII characters in meta string are not allowed"); } if (input.isEmpty()) { diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/PlatformStringUtils.java b/java/fory-core/src/main/java/org/apache/fory/serializer/PlatformStringUtils.java new file mode 100644 index 0000000000..6ac082acdc --- /dev/null +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/PlatformStringUtils.java @@ -0,0 +1,126 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.serializer; + +import java.nio.charset.StandardCharsets; +import org.apache.fory.memory.MemoryBuffer; +import org.apache.fory.platform.AndroidSupport; +import org.apache.fory.platform.GraalvmSupport; +import org.apache.fory.platform.UnsafeOps; +import org.apache.fory.platform.internal._JDKAccess; + +/** Platform-owned string internals used by {@link StringSerializer}. */ +final class PlatformStringUtils { + static final boolean JDK_STRING_FIELD_ACCESS = + !AndroidSupport.IS_ANDROID + && !GraalvmSupport.IN_GRAALVM_NATIVE_IMAGE + && _JDKAccess.JDK_STRING_FIELD_ACCESS; + static final boolean STRING_VALUE_FIELD_IS_CHARS = + JDK_STRING_FIELD_ACCESS && _JDKAccess.STRING_VALUE_FIELD_IS_CHARS; + static final boolean STRING_VALUE_FIELD_IS_BYTES = + JDK_STRING_FIELD_ACCESS && _JDKAccess.STRING_VALUE_FIELD_IS_BYTES; + static final boolean STRING_HAS_COUNT_OFFSET = + JDK_STRING_FIELD_ACCESS && _JDKAccess.STRING_HAS_COUNT_OFFSET; + + private static final long STRING_VALUE_FIELD_OFFSET = + JDK_STRING_FIELD_ACCESS ? _JDKAccess.STRING_VALUE_FIELD_OFFSET : -1; + private static final long STRING_CODER_FIELD_OFFSET = + JDK_STRING_FIELD_ACCESS ? _JDKAccess.STRING_CODER_FIELD_OFFSET : -1; + private static final long STRING_COUNT_FIELD_OFFSET = + JDK_STRING_FIELD_ACCESS ? _JDKAccess.STRING_COUNT_FIELD_OFFSET : -1; + private static final long STRING_OFFSET_FIELD_OFFSET = + JDK_STRING_FIELD_ACCESS ? _JDKAccess.STRING_OFFSET_FIELD_OFFSET : -1; + + private static final byte LATIN1 = 0; + private static final byte UTF16 = 1; + + private PlatformStringUtils() {} + + static Object getStringValue(String value) { + return UnsafeOps.getObject(value, STRING_VALUE_FIELD_OFFSET); + } + + static byte getStringCoder(String value) { + return UnsafeOps.getByte(value, STRING_CODER_FIELD_OFFSET); + } + + static int getStringOffset(String value) { + return UnsafeOps.getInt(value, STRING_OFFSET_FIELD_OFFSET); + } + + static int getStringCount(String value) { + return UnsafeOps.getInt(value, STRING_COUNT_FIELD_OFFSET); + } + + static String newCharsStringZeroCopy(char[] data) { + if (!JDK_STRING_FIELD_ACCESS) { + return new String(data); + } + return _JDKAccess.newCharsStringZeroCopy(data); + } + + static String newBytesStringZeroCopy(byte coder, byte[] data) { + if (!JDK_STRING_FIELD_ACCESS) { + return newBytesStringSlow(coder, data); + } + return _JDKAccess.newBytesStringZeroCopy(coder, data); + } + + private static String newBytesStringSlow(byte coder, byte[] data) { + if (coder == LATIN1) { + return new String(data, StandardCharsets.ISO_8859_1); + } else if (coder == UTF16) { + char[] chars = new char[data.length >> 1]; + for (int i = 0, j = 0; i < data.length; i += 2) { + chars[j++] = (char) ((data[i] & 0xff) | ((data[i + 1] & 0xff) << 8)); + } + return new String(chars); + } else { + return new String(data, StandardCharsets.UTF_8); + } + } + + static long getCharsLong(char[] chars, int charIndex) { + return UnsafeOps.getLong(chars, UnsafeOps.CHAR_ARRAY_OFFSET + ((long) charIndex << 1)); + } + + static long getBytesLong(byte[] bytes, int byteIndex) { + return UnsafeOps.getLong(bytes, UnsafeOps.BYTE_ARRAY_OFFSET + byteIndex); + } + + static char getBytesChar(byte[] bytes, int byteIndex) { + return UnsafeOps.getChar(bytes, UnsafeOps.BYTE_ARRAY_OFFSET + byteIndex); + } + + static void copyCharsToBytes( + char[] chars, int charOffset, byte[] target, int byteOffset, int numBytes) { + UnsafeOps.UNSAFE.copyMemory( + chars, + UnsafeOps.CHAR_ARRAY_OFFSET + ((long) charOffset << 1), + target, + UnsafeOps.BYTE_ARRAY_OFFSET + byteOffset, + numBytes); + } + + static void putBytes(MemoryBuffer buffer, int writerIndex, byte[] bytes, int numBytes) { + long address = buffer._unsafeWriterAddress() + writerIndex - buffer.writerIndex(); + UnsafeOps.copyMemory(bytes, UnsafeOps.BYTE_ARRAY_OFFSET, null, address, numBytes); + } +} diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/Serializers.java b/java/fory-core/src/main/java/org/apache/fory/serializer/Serializers.java index 2cca216fba..7d808985d2 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/Serializers.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/Serializers.java @@ -69,7 +69,6 @@ import org.apache.fory.serializer.scala.SingletonMapSerializer; import org.apache.fory.serializer.scala.SingletonObjectSerializer; import org.apache.fory.util.ExceptionUtils; -import org.apache.fory.util.StringUtils; /** Serialization utils and common serializers. */ @SuppressWarnings({"rawtypes", "unchecked"}) @@ -459,7 +458,7 @@ public void write(WriteContext writeContext, T value) { buffer.writeBytes(v, 0, bytesLen); } else { char[] v = (char[]) GET_VALUE.apply(value); - if (StringUtils.isLatin(v)) { + if (StringEncodingUtils.isLatin(v)) { stringSerializer.writeCharsLatin1(buffer, v, value.length()); } else { stringSerializer.writeCharsUTF16(buffer, v, value.length()); diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/SlicedStringUtil.java b/java/fory-core/src/main/java/org/apache/fory/serializer/SlicedStringUtil.java deleted file mode 100644 index d8ecbee4ce..0000000000 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/SlicedStringUtil.java +++ /dev/null @@ -1,296 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.fory.serializer; - -import org.apache.fory.memory.LittleEndian; -import org.apache.fory.memory.MemoryBuffer; -import org.apache.fory.memory.NativeByteOrder; -import org.apache.fory.platform.UnsafeOps; -import org.apache.fory.util.MathUtils; -import org.apache.fory.util.StringEncodingUtils; -import org.apache.fory.util.StringUtils; - -final class SlicedStringUtil { - private static final byte LATIN1 = 0; - private static final byte UTF16 = 1; - private static final byte UTF8 = 2; - - private SlicedStringUtil() {} - - static void writeCharsLatin1WithOffset( - StringSerializer serializer, MemoryBuffer buffer, char[] chars, int offset, int count) { - int writerIndex = buffer.writerIndex(); - long header = ((long) count << 2) | LATIN1; - buffer.ensure(writerIndex + 5 + count); - byte[] targetArray = buffer.getHeapMemory(); - if (targetArray != null) { - final int targetIndex = buffer._unsafeHeapWriterIndex(); - int arrIndex = targetIndex; - arrIndex += LittleEndian.putVarUint36Small(targetArray, arrIndex, header); - writerIndex += arrIndex - targetIndex; - for (int i = 0; i < count; i++) { - targetArray[arrIndex + i] = (byte) chars[offset + i]; - } - } else { - writerIndex += buffer._unsafePutVarUint36Small(writerIndex, header); - final byte[] tmpArray = serializer.getByteArray(count); - for (int i = 0; i < count; i++) { - tmpArray[i] = (byte) chars[offset + i]; - } - buffer.put(writerIndex, tmpArray, 0, count); - } - writerIndex += count; - buffer._unsafeWriterIndex(writerIndex); - } - - static void writeCharsUTF16WithOffset( - StringSerializer serializer, MemoryBuffer buffer, char[] chars, int offset, int count) { - int numBytes = MathUtils.doubleExact(count); - int writerIndex = buffer.writerIndex(); - long header = ((long) numBytes << 2) | UTF16; - buffer.ensure(writerIndex + 5 + numBytes); - final byte[] targetArray = buffer.getHeapMemory(); - if (targetArray != null) { - final int targetIndex = buffer._unsafeHeapWriterIndex(); - int arrIndex = targetIndex; - arrIndex += LittleEndian.putVarUint36Small(targetArray, arrIndex, header); - writerIndex += arrIndex - targetIndex + numBytes; - if (NativeByteOrder.IS_LITTLE_ENDIAN) { - // FIXME JDK11 utf16 string uses little-endian order. - UnsafeOps.UNSAFE.copyMemory( - chars, - UnsafeOps.CHAR_ARRAY_OFFSET + ((long) offset << 1), - targetArray, - UnsafeOps.BYTE_ARRAY_OFFSET + arrIndex, - numBytes); - } else { - writeCharsUTF16BEToHeap(chars, offset, arrIndex, numBytes, targetArray); - } - } else { - writerIndex += buffer._unsafePutVarUint36Small(writerIndex, header); - if (NativeByteOrder.IS_LITTLE_ENDIAN) { - writerIndex = - offHeapWriteCharsUTF16WithOffset( - serializer, buffer, chars, offset, writerIndex, numBytes); - } else { - writerIndex = - offHeapWriteCharsUTF16BEWithOffset( - serializer, buffer, chars, offset, writerIndex, numBytes); - } - } - buffer._unsafeWriterIndex(writerIndex); - } - - static void writeCharsUTF8WithOffset( - StringSerializer serializer, MemoryBuffer buffer, char[] chars, int offset, int count) { - int estimateMaxBytes = count * 3; - int approxNumBytes = (int) (count * 1.5) + 1; - int writerIndex = buffer.writerIndex(); - buffer.ensure(writerIndex + 9 + estimateMaxBytes); - byte[] targetArray = buffer.getHeapMemory(); - if (targetArray != null) { - int targetIndex = buffer._unsafeHeapWriterIndex(); - int headerPos = targetIndex; - int arrIndex = targetIndex; - long header = ((long) approxNumBytes << 2) | UTF8; - int headerBytesWritten = LittleEndian.putVarUint36Small(targetArray, arrIndex, header); - arrIndex += headerBytesWritten; - writerIndex += headerBytesWritten; - targetIndex = - StringEncodingUtils.convertUTF16ToUTF8(chars, offset, count, targetArray, arrIndex); - byte stashedByte = targetArray[arrIndex]; - int written = targetIndex - arrIndex; - header = ((long) written << 2) | UTF8; - int diff = - LittleEndian.putVarUint36Small(targetArray, headerPos, header) - headerBytesWritten; - if (diff != 0) { - handleWriteCharsUTF8UnalignedHeaderBytes(targetArray, arrIndex, diff, written, stashedByte); - } - buffer._unsafeWriterIndex(writerIndex + written + diff); - } else { - final byte[] tmpArray = serializer.getByteArray(estimateMaxBytes); - int written = StringEncodingUtils.convertUTF16ToUTF8(chars, offset, count, tmpArray, 0); - long header = ((long) written << 2) | UTF8; - writerIndex += buffer._unsafePutVarUint36Small(writerIndex, header); - buffer.put(writerIndex, tmpArray, 0, written); - buffer._unsafeWriterIndex(writerIndex + written); - } - } - - static void writeCharsUTF8PerfOptimizedWithOffset( - StringSerializer serializer, MemoryBuffer buffer, char[] chars, int offset, int count) { - int estimateMaxBytes = count * 3; - int numBytes = MathUtils.doubleExact(count); - int writerIndex = buffer.writerIndex(); - long header = ((long) numBytes << 2) | UTF8; - buffer.ensure(writerIndex + 9 + estimateMaxBytes); - byte[] targetArray = buffer.getHeapMemory(); - if (targetArray != null) { - int targetIndex = buffer._unsafeHeapWriterIndex(); - int arrIndex = targetIndex; - arrIndex += LittleEndian.putVarUint36Small(targetArray, arrIndex, header); - writerIndex += arrIndex - targetIndex; - targetIndex = - StringEncodingUtils.convertUTF16ToUTF8(chars, offset, count, targetArray, arrIndex + 4); - int written = targetIndex - arrIndex - 4; - buffer._unsafePutInt32(writerIndex, written); - buffer._unsafeWriterIndex(writerIndex + 4 + written); - } else { - final byte[] tmpArray = serializer.getByteArray(estimateMaxBytes); - int written = StringEncodingUtils.convertUTF16ToUTF8(chars, offset, count, tmpArray, 0); - writerIndex += buffer._unsafePutVarUint36Small(writerIndex, header); - buffer._unsafePutInt32(writerIndex, written); - writerIndex += 4; - buffer.put(writerIndex, tmpArray, 0, written); - buffer._unsafeWriterIndex(writerIndex + written); - } - } - - static boolean isLatin(char[] chars, int offset, int count) { - int end = offset + count; - int vectorizedChars = count & ~3; - int vectorEnd = offset + vectorizedChars; - long byteOffset = UnsafeOps.CHAR_ARRAY_OFFSET + ((long) offset << 1); - long endOffset = UnsafeOps.CHAR_ARRAY_OFFSET + ((long) vectorEnd << 1); - for (long off = byteOffset; off < endOffset; off += 8) { - long multiChars = UnsafeOps.getLong(chars, off); - if ((multiChars & StringUtils.MULTI_CHARS_NON_LATIN_MASK) != 0) { - return false; - } - } - for (int i = vectorEnd; i < end; i++) { - if (chars[i] > 0xFF) { - return false; - } - } - return true; - } - - static byte bestCoder(char[] chars, int offset, int count) { - int sampleNum = Math.min(64, count); - int vectorizedLen = sampleNum >> 2; - int vectorizedChars = vectorizedLen << 2; - long byteOffset = UnsafeOps.CHAR_ARRAY_OFFSET + ((long) offset << 1); - long endOffset = byteOffset + ((long) vectorizedChars << 1); - int asciiCount = 0; - int latin1Count = 0; - int charOffset = offset; - for (long off = byteOffset; off < endOffset; off += 8, charOffset += 4) { - long multiChars = UnsafeOps.getLong(chars, off); - if ((multiChars & StringUtils.MULTI_CHARS_NON_ASCII_MASK) == 0) { - latin1Count += 4; - asciiCount += 4; - } else if ((multiChars & StringUtils.MULTI_CHARS_NON_LATIN_MASK) == 0) { - latin1Count += 4; - for (int i = 0; i < 4; ++i) { - if (chars[charOffset + i] < 0x80) { - asciiCount++; - } - } - } else { - for (int i = 0; i < 4; ++i) { - char c = chars[charOffset + i]; - if (c < 0x80) { - latin1Count++; - asciiCount++; - } else if (c <= 0xFF) { - latin1Count++; - } - } - } - } - - for (int i = vectorizedChars; i < sampleNum; i++) { - char c = chars[offset + i]; - if (c < 0x80) { - latin1Count++; - asciiCount++; - } else if (c <= 0xFF) { - latin1Count++; - } - } - - if (latin1Count == count || (latin1Count == sampleNum && isLatin(chars, offset, count))) { - return LATIN1; - } else if (asciiCount >= sampleNum * 0.5) { - return UTF8; - } else { - return UTF16; - } - } - - private static void handleWriteCharsUTF8UnalignedHeaderBytes( - byte[] targetArray, int arrIndex, int diff, int written, byte stashed) { - if (diff == 1) { - System.arraycopy(targetArray, arrIndex + 1, targetArray, arrIndex + 2, written - 1); - targetArray[arrIndex + 1] = stashed; - } else { - System.arraycopy(targetArray, arrIndex, targetArray, arrIndex - 1, written); - } - } - - private static void writeCharsUTF16BEToHeap( - char[] chars, int offset, int arrIndex, int numBytes, byte[] targetArray) { - int charIndex = offset; - for (int i = arrIndex, end = i + numBytes; i < end; i += 2) { - char c = chars[charIndex++]; - targetArray[i] = (byte) c; - targetArray[i + 1] = (byte) (c >>> 8); - } - } - - private static int offHeapWriteCharsUTF16WithOffset( - StringSerializer serializer, - MemoryBuffer buffer, - char[] chars, - int offset, - int writerIndex, - int numBytes) { - byte[] tmpArray = serializer.getByteArray(numBytes); - UnsafeOps.UNSAFE.copyMemory( - chars, - UnsafeOps.CHAR_ARRAY_OFFSET + ((long) offset << 1), - tmpArray, - UnsafeOps.BYTE_ARRAY_OFFSET, - numBytes); - buffer.put(writerIndex, tmpArray, 0, numBytes); - writerIndex += numBytes; - return writerIndex; - } - - private static int offHeapWriteCharsUTF16BEWithOffset( - StringSerializer serializer, - MemoryBuffer buffer, - char[] chars, - int offset, - int writerIndex, - int numBytes) { - byte[] tmpArray = serializer.getByteArray(numBytes); - int charIndex = offset; - for (int i = 0; i < numBytes; i += 2) { - char c = chars[charIndex++]; - tmpArray[i] = (byte) c; - tmpArray[i + 1] = (byte) (c >>> 8); - } - buffer.put(writerIndex, tmpArray, 0, numBytes); - writerIndex += numBytes; - return writerIndex; - } -} diff --git a/java/fory-core/src/main/java/org/apache/fory/util/StringEncodingUtils.java b/java/fory-core/src/main/java/org/apache/fory/serializer/StringEncodingUtils.java similarity index 59% rename from java/fory-core/src/main/java/org/apache/fory/util/StringEncodingUtils.java rename to java/fory-core/src/main/java/org/apache/fory/serializer/StringEncodingUtils.java index e409dd3bff..186d4e2455 100644 --- a/java/fory-core/src/main/java/org/apache/fory/util/StringEncodingUtils.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/StringEncodingUtils.java @@ -17,22 +17,73 @@ * under the License. */ -package org.apache.fory.util; +package org.apache.fory.serializer; import static org.apache.fory.util.StringUtils.MULTI_CHARS_NON_ASCII_MASK; +import static org.apache.fory.util.StringUtils.MULTI_CHARS_NON_LATIN_MASK; +import org.apache.fory.memory.LittleEndian; +import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.memory.NativeByteOrder; -import org.apache.fory.platform.UnsafeOps; +import org.apache.fory.util.MathUtils; /** String Encoding Utils. */ public class StringEncodingUtils { + private static final byte LATIN1 = 0; + private static final byte UTF16 = 1; + private static final byte UTF8 = 2; + + public static boolean isLatin(char[] chars) { + return isLatin(chars, 0); + } + + public static boolean isLatin(char[] chars, int start) { + if (start > chars.length) { + return false; + } + int numChars = chars.length; + int charIndex = start; + while (charIndex + 4 <= numChars) { + // Check 4 chars in a vectorized way. See CompressStringSuite.latinSuperWordCheck. + long multiChars = PlatformStringUtils.getCharsLong(chars, charIndex); + if ((multiChars & MULTI_CHARS_NON_LATIN_MASK) != 0) { + return false; + } + charIndex += 4; + } + for (int i = charIndex; i < numChars; i++) { + if (chars[i] > 0xFF) { + return false; + } + } + return true; + } + + static boolean isLatin(char[] chars, int offset, int count) { + int end = offset + count; + int vectorizedChars = count & ~3; + int vectorEnd = offset + vectorizedChars; + for (int charIndex = offset; charIndex < vectorEnd; charIndex += 4) { + long multiChars = PlatformStringUtils.getCharsLong(chars, charIndex); + if ((multiChars & MULTI_CHARS_NON_LATIN_MASK) != 0) { + return false; + } + } + for (int i = vectorEnd; i < end; i++) { + if (chars[i] > 0xFF) { + return false; + } + } + return true; + } /** A fast convert algorithm to convert an utf16 char array into an utf8 byte array. */ public static int convertUTF16ToUTF8(char[] src, byte[] dst, int dp) { int numChars = src.length; - for (int charOffset = 0, arrayOffset = UnsafeOps.CHAR_ARRAY_OFFSET; charOffset < numChars; ) { + for (int charOffset = 0; charOffset < numChars; ) { if (charOffset + 4 <= numChars - && (UnsafeOps.getLong(src, arrayOffset) & MULTI_CHARS_NON_ASCII_MASK) == 0) { + && (PlatformStringUtils.getCharsLong(src, charOffset) & MULTI_CHARS_NON_ASCII_MASK) + == 0) { // ascii only dst[dp] = (byte) src[charOffset]; dst[dp + 1] = (byte) src[charOffset + 1]; @@ -40,10 +91,8 @@ public static int convertUTF16ToUTF8(char[] src, byte[] dst, int dp) { dst[dp + 3] = (byte) src[charOffset + 3]; dp += 4; charOffset += 4; - arrayOffset += 8; } else { char c = src[charOffset++]; - arrayOffset += 2; if (c < 0x80) { dst[dp++] = (byte) c; } else if (c < 0x800) { @@ -54,7 +103,6 @@ public static int convertUTF16ToUTF8(char[] src, byte[] dst, int dp) { utf8ToChar2(src, charOffset, c, dst, dp); dp += 4; charOffset++; - arrayOffset += 2; } else { dst[dp] = (byte) (0xe0 | ((c >> 12))); dst[dp + 1] = (byte) (0x80 | ((c >> 6) & 0x3f)); @@ -69,20 +117,18 @@ public static int convertUTF16ToUTF8(char[] src, byte[] dst, int dp) { /** A fast convert algorithm to convert an utf16 char array slice into an utf8 byte array. */ public static int convertUTF16ToUTF8(char[] src, int offset, int len, byte[] dst, int dp) { int end = offset + len; - for (int charOffset = offset, arrayOffset = UnsafeOps.CHAR_ARRAY_OFFSET + (offset << 1); - charOffset < end; ) { + for (int charOffset = offset; charOffset < end; ) { if (charOffset + 4 <= end - && (UnsafeOps.getLong(src, arrayOffset) & MULTI_CHARS_NON_ASCII_MASK) == 0) { + && (PlatformStringUtils.getCharsLong(src, charOffset) & MULTI_CHARS_NON_ASCII_MASK) + == 0) { dst[dp] = (byte) src[charOffset]; dst[dp + 1] = (byte) src[charOffset + 1]; dst[dp + 2] = (byte) src[charOffset + 2]; dst[dp + 3] = (byte) src[charOffset + 3]; dp += 4; charOffset += 4; - arrayOffset += 8; } else { char c = src[charOffset++]; - arrayOffset += 2; if (c < 0x80) { dst[dp++] = (byte) c; } else if (c < 0x800) { @@ -96,7 +142,6 @@ public static int convertUTF16ToUTF8(char[] src, int offset, int len, byte[] dst utf8ToChar2(src, charOffset, c, dst, dp); dp += 4; charOffset++; - arrayOffset += 2; } else { dst[dp] = (byte) (0xe0 | ((c >> 12))); dst[dp + 1] = (byte) (0x80 | ((c >> 6) & 0x3f)); @@ -113,9 +158,7 @@ public static int convertUTF16ToUTF8(byte[] src, byte[] dst, int dp) { int numBytes = src.length; for (int offset = 0; offset < numBytes; ) { if (offset + 8 <= numBytes - && (UnsafeOps.getLong(src, UnsafeOps.BYTE_ARRAY_OFFSET + offset) - & MULTI_CHARS_NON_ASCII_MASK) - == 0) { + && (PlatformStringUtils.getBytesLong(src, offset) & MULTI_CHARS_NON_ASCII_MASK) == 0) { // ascii only if (NativeByteOrder.IS_LITTLE_ENDIAN) { dst[dp] = src[offset]; @@ -131,7 +174,7 @@ public static int convertUTF16ToUTF8(byte[] src, byte[] dst, int dp) { dp += 4; offset += 8; } else { - char c = UnsafeOps.getChar(src, UnsafeOps.BYTE_ARRAY_OFFSET + offset); + char c = PlatformStringUtils.getBytesChar(src, offset); offset += 2; if (c < 0x80) { @@ -169,8 +212,7 @@ public static int convertUTF8ToUTF16(byte[] src, int offset, int len, byte[] dst while (offset < end) { if (offset + 8 <= end - && (UnsafeOps.getLong(src, UnsafeOps.BYTE_ARRAY_OFFSET + offset) & 0x8080808080808080L) - == 0) { + && (PlatformStringUtils.getBytesLong(src, offset) & 0x8080808080808080L) == 0) { // ascii only if (NativeByteOrder.IS_LITTLE_ENDIAN) { dst[dp] = src[offset]; @@ -292,8 +334,7 @@ public static int convertUTF8ToUTF16(byte[] src, int offset, int len, char[] dst int dp = 0; while (offset < end) { if (offset + 8 <= end - && (UnsafeOps.getLong(src, UnsafeOps.BYTE_ARRAY_OFFSET + offset) & 0x8080808080808080L) - == 0) { + && (PlatformStringUtils.getBytesLong(src, offset) & 0x8080808080808080L) == 0) { // ascii only dst[dp] = (char) src[offset]; dst[dp + 1] = (char) src[offset + 1]; @@ -410,8 +451,7 @@ private static void utf8ToChar2( char d; if (c > Character.MAX_HIGH_SURROGATE || numBytes - offset < 1 - || (d = UnsafeOps.getChar(src, UnsafeOps.BYTE_ARRAY_OFFSET + offset)) - < Character.MIN_LOW_SURROGATE + || (d = PlatformStringUtils.getBytesChar(src, offset)) < Character.MIN_LOW_SURROGATE || d > Character.MAX_LOW_SURROGATE) { throw new RuntimeException("malformed input off : " + offset); } @@ -466,8 +506,7 @@ public static int convertUTF8ToLatin1(byte[] src, int offset, int length, byte[] while (offset < end) { // Vectorized ASCII fast path if (offset + 8 <= end - && (UnsafeOps.getLong(src, UnsafeOps.BYTE_ARRAY_OFFSET + offset) & 0x8080808080808080L) - == 0) { + && (PlatformStringUtils.getBytesLong(src, offset) & 0x8080808080808080L) == 0) { // 8 ASCII bytes - direct copy dst[dstPos] = src[offset]; dst[dstPos + 1] = src[offset + 1]; @@ -508,7 +547,7 @@ public static boolean isUTF8WithinAscii(byte[] bytes, int offset, int length) { // Check 8 bytes at a time for (int i = offset; i < vectorizedEnd; i += 8) { - if ((UnsafeOps.getLong(bytes, UnsafeOps.BYTE_ARRAY_OFFSET + i) & 0x8080808080808080L) != 0) { + if ((PlatformStringUtils.getBytesLong(bytes, i) & 0x8080808080808080L) != 0) { return false; } } @@ -522,4 +561,231 @@ public static boolean isUTF8WithinAscii(byte[] bytes, int offset, int length) { return true; } + + static void writeCharsLatin1WithOffset( + StringSerializer serializer, MemoryBuffer buffer, char[] chars, int offset, int count) { + int writerIndex = buffer.writerIndex(); + long header = ((long) count << 2) | LATIN1; + buffer.ensure(writerIndex + 5 + count); + byte[] targetArray = buffer.getHeapMemory(); + if (targetArray != null) { + final int targetIndex = buffer._unsafeHeapWriterIndex(); + int arrIndex = targetIndex; + arrIndex += LittleEndian.putVarUint36Small(targetArray, arrIndex, header); + writerIndex += arrIndex - targetIndex; + for (int i = 0; i < count; i++) { + targetArray[arrIndex + i] = (byte) chars[offset + i]; + } + } else { + writerIndex += buffer._unsafePutVarUint36Small(writerIndex, header); + final byte[] tmpArray = serializer.getByteArray(count); + for (int i = 0; i < count; i++) { + tmpArray[i] = (byte) chars[offset + i]; + } + buffer.put(writerIndex, tmpArray, 0, count); + } + writerIndex += count; + buffer._unsafeWriterIndex(writerIndex); + } + + static void writeCharsUTF16WithOffset( + StringSerializer serializer, MemoryBuffer buffer, char[] chars, int offset, int count) { + int numBytes = MathUtils.doubleExact(count); + int writerIndex = buffer.writerIndex(); + long header = ((long) numBytes << 2) | UTF16; + buffer.ensure(writerIndex + 5 + numBytes); + final byte[] targetArray = buffer.getHeapMemory(); + if (targetArray != null) { + final int targetIndex = buffer._unsafeHeapWriterIndex(); + int arrIndex = targetIndex; + arrIndex += LittleEndian.putVarUint36Small(targetArray, arrIndex, header); + writerIndex += arrIndex - targetIndex + numBytes; + if (NativeByteOrder.IS_LITTLE_ENDIAN) { + // FIXME JDK11 utf16 string uses little-endian order. + PlatformStringUtils.copyCharsToBytes(chars, offset, targetArray, arrIndex, numBytes); + } else { + writeCharsUTF16BEToHeap(chars, offset, arrIndex, numBytes, targetArray); + } + } else { + writerIndex += buffer._unsafePutVarUint36Small(writerIndex, header); + if (NativeByteOrder.IS_LITTLE_ENDIAN) { + writerIndex = + offHeapWriteCharsUTF16WithOffset( + serializer, buffer, chars, offset, writerIndex, numBytes); + } else { + writerIndex = + offHeapWriteCharsUTF16BEWithOffset( + serializer, buffer, chars, offset, writerIndex, numBytes); + } + } + buffer._unsafeWriterIndex(writerIndex); + } + + static void writeCharsUTF8WithOffset( + StringSerializer serializer, MemoryBuffer buffer, char[] chars, int offset, int count) { + int estimateMaxBytes = count * 3; + int approxNumBytes = (int) (count * 1.5) + 1; + int writerIndex = buffer.writerIndex(); + buffer.ensure(writerIndex + 9 + estimateMaxBytes); + byte[] targetArray = buffer.getHeapMemory(); + if (targetArray != null) { + int targetIndex = buffer._unsafeHeapWriterIndex(); + int headerPos = targetIndex; + int arrIndex = targetIndex; + long header = ((long) approxNumBytes << 2) | UTF8; + int headerBytesWritten = LittleEndian.putVarUint36Small(targetArray, arrIndex, header); + arrIndex += headerBytesWritten; + writerIndex += headerBytesWritten; + targetIndex = convertUTF16ToUTF8(chars, offset, count, targetArray, arrIndex); + byte stashedByte = targetArray[arrIndex]; + int written = targetIndex - arrIndex; + header = ((long) written << 2) | UTF8; + int diff = + LittleEndian.putVarUint36Small(targetArray, headerPos, header) - headerBytesWritten; + if (diff != 0) { + handleWriteCharsUTF8UnalignedHeaderBytes(targetArray, arrIndex, diff, written, stashedByte); + } + buffer._unsafeWriterIndex(writerIndex + written + diff); + } else { + final byte[] tmpArray = serializer.getByteArray(estimateMaxBytes); + int written = convertUTF16ToUTF8(chars, offset, count, tmpArray, 0); + long header = ((long) written << 2) | UTF8; + writerIndex += buffer._unsafePutVarUint36Small(writerIndex, header); + buffer.put(writerIndex, tmpArray, 0, written); + buffer._unsafeWriterIndex(writerIndex + written); + } + } + + static void writeCharsUTF8PerfOptimizedWithOffset( + StringSerializer serializer, MemoryBuffer buffer, char[] chars, int offset, int count) { + int estimateMaxBytes = count * 3; + int numBytes = MathUtils.doubleExact(count); + int writerIndex = buffer.writerIndex(); + long header = ((long) numBytes << 2) | UTF8; + buffer.ensure(writerIndex + 9 + estimateMaxBytes); + byte[] targetArray = buffer.getHeapMemory(); + if (targetArray != null) { + int targetIndex = buffer._unsafeHeapWriterIndex(); + int arrIndex = targetIndex; + arrIndex += LittleEndian.putVarUint36Small(targetArray, arrIndex, header); + writerIndex += arrIndex - targetIndex; + targetIndex = convertUTF16ToUTF8(chars, offset, count, targetArray, arrIndex + 4); + int written = targetIndex - arrIndex - 4; + buffer._unsafePutInt32(writerIndex, written); + buffer._unsafeWriterIndex(writerIndex + 4 + written); + } else { + final byte[] tmpArray = serializer.getByteArray(estimateMaxBytes); + int written = convertUTF16ToUTF8(chars, offset, count, tmpArray, 0); + writerIndex += buffer._unsafePutVarUint36Small(writerIndex, header); + buffer._unsafePutInt32(writerIndex, written); + writerIndex += 4; + buffer.put(writerIndex, tmpArray, 0, written); + buffer._unsafeWriterIndex(writerIndex + written); + } + } + + static byte bestCoder(char[] chars, int offset, int count) { + int sampleNum = Math.min(64, count); + int vectorizedLen = sampleNum >> 2; + int vectorizedChars = vectorizedLen << 2; + int asciiCount = 0; + int latin1Count = 0; + int charOffset = offset; + int vectorEnd = offset + vectorizedChars; + for (; charOffset < vectorEnd; charOffset += 4) { + long multiChars = PlatformStringUtils.getCharsLong(chars, charOffset); + if ((multiChars & MULTI_CHARS_NON_ASCII_MASK) == 0) { + latin1Count += 4; + asciiCount += 4; + } else if ((multiChars & MULTI_CHARS_NON_LATIN_MASK) == 0) { + latin1Count += 4; + for (int i = 0; i < 4; ++i) { + if (chars[charOffset + i] < 0x80) { + asciiCount++; + } + } + } else { + for (int i = 0; i < 4; ++i) { + char c = chars[charOffset + i]; + if (c < 0x80) { + latin1Count++; + asciiCount++; + } else if (c <= 0xFF) { + latin1Count++; + } + } + } + } + + for (int i = vectorizedChars; i < sampleNum; i++) { + char c = chars[offset + i]; + if (c < 0x80) { + latin1Count++; + asciiCount++; + } else if (c <= 0xFF) { + latin1Count++; + } + } + + if (latin1Count == count || (latin1Count == sampleNum && isLatin(chars, offset, count))) { + return LATIN1; + } else if (asciiCount >= sampleNum * 0.5) { + return UTF8; + } else { + return UTF16; + } + } + + private static void handleWriteCharsUTF8UnalignedHeaderBytes( + byte[] targetArray, int arrIndex, int diff, int written, byte stashed) { + if (diff == 1) { + System.arraycopy(targetArray, arrIndex + 1, targetArray, arrIndex + 2, written - 1); + targetArray[arrIndex + 1] = stashed; + } else { + System.arraycopy(targetArray, arrIndex, targetArray, arrIndex - 1, written); + } + } + + private static void writeCharsUTF16BEToHeap( + char[] chars, int offset, int arrIndex, int numBytes, byte[] targetArray) { + int charIndex = offset; + for (int i = arrIndex, end = i + numBytes; i < end; i += 2) { + char c = chars[charIndex++]; + targetArray[i] = (byte) c; + targetArray[i + 1] = (byte) (c >>> 8); + } + } + + private static int offHeapWriteCharsUTF16WithOffset( + StringSerializer serializer, + MemoryBuffer buffer, + char[] chars, + int offset, + int writerIndex, + int numBytes) { + byte[] tmpArray = serializer.getByteArray(numBytes); + PlatformStringUtils.copyCharsToBytes(chars, offset, tmpArray, 0, numBytes); + buffer.put(writerIndex, tmpArray, 0, numBytes); + writerIndex += numBytes; + return writerIndex; + } + + private static int offHeapWriteCharsUTF16BEWithOffset( + StringSerializer serializer, + MemoryBuffer buffer, + char[] chars, + int offset, + int writerIndex, + int numBytes) { + byte[] tmpArray = serializer.getByteArray(numBytes); + int charIndex = offset; + for (int i = 0; i < numBytes; i += 2) { + char c = chars[charIndex++]; + tmpArray[i] = (byte) c; + tmpArray[i + 1] = (byte) (c >>> 8); + } + buffer.put(writerIndex, tmpArray, 0, numBytes); + writerIndex += numBytes; + return writerIndex; + } } diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/StringSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/StringSerializer.java index 8dafbccdd9..9e4f75d3df 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/StringSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/StringSerializer.java @@ -36,13 +36,8 @@ import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.memory.NativeByteOrder; import org.apache.fory.platform.AndroidSupport; -import org.apache.fory.platform.GraalvmSupport; -import org.apache.fory.platform.UnsafeOps; -import org.apache.fory.platform.internal._JDKAccess; import org.apache.fory.util.MathUtils; import org.apache.fory.util.Preconditions; -import org.apache.fory.util.StringEncodingUtils; -import org.apache.fory.util.StringUtils; /** * String serializer based on JDK-internal string access and byte-array accessors for speed. @@ -53,43 +48,21 @@ */ @SuppressWarnings("unchecked") public final class StringSerializer extends ImmutableSerializer { - private static final boolean STRING_VALUE_FIELD_IS_CHARS; - private static final boolean STRING_VALUE_FIELD_IS_BYTES; + private static final boolean STRING_VALUE_FIELD_IS_CHARS = + PlatformStringUtils.STRING_VALUE_FIELD_IS_CHARS; + private static final boolean STRING_VALUE_FIELD_IS_BYTES = + PlatformStringUtils.STRING_VALUE_FIELD_IS_BYTES; private static final byte LATIN1 = 0; private static final byte UTF16 = 1; private static final byte UTF8 = 2; private static final int DEFAULT_BUFFER_SIZE = 1024; - private static final long STRING_VALUE_FIELD_OFFSET; - private static final boolean STRING_HAS_COUNT_OFFSET; - private static final long STRING_COUNT_FIELD_OFFSET; - private static final long STRING_OFFSET_FIELD_OFFSET; - - static { - if (!jdkInternalFieldAccess()) { - STRING_VALUE_FIELD_IS_CHARS = false; - STRING_VALUE_FIELD_IS_BYTES = false; - STRING_VALUE_FIELD_OFFSET = -1; - STRING_HAS_COUNT_OFFSET = false; - STRING_COUNT_FIELD_OFFSET = -1; - STRING_OFFSET_FIELD_OFFSET = -1; - } else { - STRING_VALUE_FIELD_IS_CHARS = _JDKAccess.STRING_VALUE_FIELD_IS_CHARS; - STRING_VALUE_FIELD_IS_BYTES = _JDKAccess.STRING_VALUE_FIELD_IS_BYTES; - STRING_VALUE_FIELD_OFFSET = _JDKAccess.STRING_VALUE_FIELD_OFFSET; - STRING_HAS_COUNT_OFFSET = _JDKAccess.STRING_HAS_COUNT_OFFSET; - STRING_COUNT_FIELD_OFFSET = _JDKAccess.STRING_COUNT_FIELD_OFFSET; - STRING_OFFSET_FIELD_OFFSET = _JDKAccess.STRING_OFFSET_FIELD_OFFSET; - } - } + private static final boolean STRING_HAS_COUNT_OFFSET = + PlatformStringUtils.STRING_HAS_COUNT_OFFSET; private static boolean jdkInternalFieldAccess() { - // Native-image runtime string layout is not a HotSpot Unsafe-offset contract; use public - // string copies there even when the image build can see JDK private fields. - return !AndroidSupport.IS_ANDROID - && !GraalvmSupport.IN_GRAALVM_NATIVE_IMAGE - && _JDKAccess.JDK_STRING_FIELD_ACCESS; + return PlatformStringUtils.JDK_STRING_FIELD_ACCESS; } private final boolean compressString; @@ -380,7 +353,7 @@ public String readString(MemoryBuffer buffer) { private void writeStringSlow(MemoryBuffer buffer, String value) { char[] chars = value.toCharArray(); - if (isLatin(chars)) { + if (StringEncodingUtils.isLatin(chars)) { writeCharsLatin1(buffer, chars, chars.length); return; } @@ -427,29 +400,20 @@ private static void writeVarUint36Small(MemoryBuffer buffer, long value) { buffer._unsafeWriterIndex(writerIndex); } - private static boolean isLatin(char[] chars) { - for (char c : chars) { - if (c > 0xFF) { - return false; - } - } - return true; - } - private static Object getStringValue(String value) { - return UnsafeOps.getObject(value, STRING_VALUE_FIELD_OFFSET); + return PlatformStringUtils.getStringValue(value); } private static byte getStringCoder(String value) { - return UnsafeOps.getByte(value, _JDKAccess.STRING_CODER_FIELD_OFFSET); + return PlatformStringUtils.getStringCoder(value); } private static int getStringOffset(String value) { - return UnsafeOps.getInt(value, STRING_OFFSET_FIELD_OFFSET); + return PlatformStringUtils.getStringOffset(value); } private static int getStringCount(String value) { - return UnsafeOps.getInt(value, STRING_COUNT_FIELD_OFFSET); + return PlatformStringUtils.getStringCount(value); } @CodegenInvoke @@ -489,17 +453,18 @@ public void writeCompressedCharsStringWithOffset(MemoryBuffer buffer, String val final char[] chars = (char[]) getStringValue(value); final int offset = getStringOffset(value); final int count = getStringCount(value); - final byte coder = SlicedStringUtil.bestCoder(chars, offset, count); + final byte coder = StringEncodingUtils.bestCoder(chars, offset, count); if (coder == LATIN1) { - SlicedStringUtil.writeCharsLatin1WithOffset(this, buffer, chars, offset, count); + StringEncodingUtils.writeCharsLatin1WithOffset(this, buffer, chars, offset, count); } else if (coder == UTF8) { if (writeNumUtf16BytesForUtf8Encoding) { - SlicedStringUtil.writeCharsUTF8PerfOptimizedWithOffset(this, buffer, chars, offset, count); + StringEncodingUtils.writeCharsUTF8PerfOptimizedWithOffset( + this, buffer, chars, offset, count); } else { - SlicedStringUtil.writeCharsUTF8WithOffset(this, buffer, chars, offset, count); + StringEncodingUtils.writeCharsUTF8WithOffset(this, buffer, chars, offset, count); } } else { - SlicedStringUtil.writeCharsUTF16WithOffset(this, buffer, chars, offset, count); + StringEncodingUtils.writeCharsUTF16WithOffset(this, buffer, chars, offset, count); } } @@ -531,14 +496,8 @@ public static void writeBytesString(MemoryBuffer buffer, byte coder, byte[] byte writerIndex += arrIndex - targetIndex; System.arraycopy(bytes, 0, targetArray, arrIndex, bytesLen); } else { - final long targetAddress = buffer._unsafeWriterAddress(); - final int headerBytes = buffer._unsafePutVarUint36Small(writerIndex, header); - writerIndex += headerBytes; - // The preceding ensure makes this copy bounds-safe. Keep the direct copy local so generated - // string serializers do not route every off-heap string payload through checked raw-copy - // helpers. - UnsafeOps.copyMemory( - bytes, UnsafeOps.BYTE_ARRAY_OFFSET, null, targetAddress + headerBytes, bytesLen); + writerIndex += buffer._unsafePutVarUint36Small(writerIndex, header); + PlatformStringUtils.putBytes(buffer, writerIndex, bytes, bytesLen); } writerIndex += bytesLen; buffer._unsafeWriterIndex(writerIndex); @@ -547,7 +506,7 @@ public static void writeBytesString(MemoryBuffer buffer, byte coder, byte[] byte @CodegenInvoke public void writeCharsString(MemoryBuffer buffer, String value) { final char[] chars = (char[]) getStringValue(value); - if (StringUtils.isLatin(chars)) { + if (StringEncodingUtils.isLatin(chars)) { writeCharsLatin1(buffer, chars, chars.length); } else { writeCharsUTF16(buffer, chars, chars.length); @@ -559,10 +518,10 @@ public void writeCharsStringWithOffset(MemoryBuffer buffer, String value) { final char[] chars = (char[]) getStringValue(value); final int offset = getStringOffset(value); final int count = getStringCount(value); - if (SlicedStringUtil.isLatin(chars, offset, count)) { - SlicedStringUtil.writeCharsLatin1WithOffset(this, buffer, chars, offset, count); + if (StringEncodingUtils.isLatin(chars, offset, count)) { + StringEncodingUtils.writeCharsLatin1WithOffset(this, buffer, chars, offset, count); } else { - SlicedStringUtil.writeCharsUTF16WithOffset(this, buffer, chars, offset, count); + StringEncodingUtils.writeCharsUTF16WithOffset(this, buffer, chars, offset, count); } } @@ -729,12 +688,7 @@ public void writeCharsUTF16(MemoryBuffer buffer, char[] chars, int numChars) { writeCharsUTF16ToHeapSlow(chars, arrIndex, numBytes, targetArray); } else { // FIXME JDK11 utf16 string uses little-endian order. - UnsafeOps.UNSAFE.copyMemory( - chars, - UnsafeOps.CHAR_ARRAY_OFFSET, - targetArray, - UnsafeOps.BYTE_ARRAY_OFFSET + arrIndex, - numBytes); + PlatformStringUtils.copyCharsToBytes(chars, 0, targetArray, arrIndex, numBytes); } } else { writeCharsUTF16BEToHeap(chars, arrIndex, numBytes, targetArray); @@ -896,37 +850,13 @@ private void writeBytesUTF8PerfOptimized(MemoryBuffer buffer, byte[] bytes) { } public static String newCharsStringZeroCopy(char[] data) { - if (!jdkInternalFieldAccess()) { - return newCharsStringSlow(data); - } - return _JDKAccess.newCharsStringZeroCopy(data); - } - - private static String newCharsStringSlow(char[] data) { - return new String(data); + return PlatformStringUtils.newCharsStringZeroCopy(data); } // coder param first to make inline call args // `(buffer.readByte(), buffer.readBytesWithSizeEmbedded())` work. public static String newBytesStringZeroCopy(byte coder, byte[] data) { - if (!jdkInternalFieldAccess()) { - return newBytesStringSlow(coder, data); - } - return _JDKAccess.newBytesStringZeroCopy(coder, data); - } - - private static String newBytesStringSlow(byte coder, byte[] data) { - if (coder == LATIN1) { - return new String(data, StandardCharsets.ISO_8859_1); - } else if (coder == UTF16) { - char[] chars = new char[data.length >> 1]; - for (int i = 0, j = 0; i < data.length; i += 2) { - chars[j++] = (char) ((data[i] & 0xff) | ((data[i + 1] & 0xff) << 8)); - } - return new String(chars); - } else { - return new String(data, StandardCharsets.UTF_8); - } + return PlatformStringUtils.newBytesStringZeroCopy(coder, data); } private static void writeCharsUTF16BEToHeap( @@ -1045,13 +975,10 @@ private static byte bestCoder(char[] chars) { int sampleNum = Math.min(64, numChars); int vectorizedLen = sampleNum >> 2; int vectorizedChars = vectorizedLen << 2; - int endOffset = UnsafeOps.CHAR_ARRAY_OFFSET + (vectorizedChars << 1); int asciiCount = 0; int latin1Count = 0; - for (int offset = UnsafeOps.CHAR_ARRAY_OFFSET, charOffset = 0; - offset < endOffset; - offset += 8, charOffset += 4) { - long multiChars = UnsafeOps.getLong(chars, offset); + for (int charOffset = 0; charOffset < vectorizedChars; charOffset += 4) { + long multiChars = PlatformStringUtils.getCharsLong(chars, charOffset); if ((multiChars & MULTI_CHARS_NON_ASCII_MASK) == 0) { latin1Count += 4; asciiCount += 4; @@ -1084,7 +1011,7 @@ private static byte bestCoder(char[] chars) { } if (latin1Count == numChars - || (latin1Count == sampleNum && StringUtils.isLatin(chars, sampleNum))) { + || (latin1Count == sampleNum && StringEncodingUtils.isLatin(chars, sampleNum))) { return LATIN1; } else if (asciiCount >= sampleNum * 0.5) { // ascii number > 50%, choose UTF-8 @@ -1100,24 +1027,21 @@ private static byte bestCoder(byte[] bytes) { int sampleNum = Math.min(64 << 1, numBytes); int vectorizedLen = sampleNum >> 3; int vectorizedBytes = vectorizedLen << 3; - int endOffset = UnsafeOps.BYTE_ARRAY_OFFSET + vectorizedBytes; int asciiCount = 0; - for (int offset = UnsafeOps.BYTE_ARRAY_OFFSET, bytesOffset = 0; - offset < endOffset; - offset += 8, bytesOffset += 8) { - long multiChars = UnsafeOps.getLong(bytes, offset); + for (int bytesOffset = 0; bytesOffset < vectorizedBytes; bytesOffset += 8) { + long multiChars = PlatformStringUtils.getBytesLong(bytes, bytesOffset); if ((multiChars & MULTI_CHARS_NON_ASCII_MASK) == 0) { asciiCount += 4; } else { for (int i = 0; i < 8; i += 2) { - if (UnsafeOps.getChar(bytes, offset + i) < 0x80) { + if (PlatformStringUtils.getBytesChar(bytes, bytesOffset + i) < 0x80) { asciiCount++; } } } } for (int i = vectorizedBytes; vectorizedBytes < sampleNum; vectorizedBytes += 2) { - if (UnsafeOps.getChar(bytes, UnsafeOps.BYTE_ARRAY_OFFSET + i) < 0x80) { + if (PlatformStringUtils.getBytesChar(bytes, i) < 0x80) { asciiCount++; } } diff --git a/java/fory-core/src/main/java/org/apache/fory/util/StringUtils.java b/java/fory-core/src/main/java/org/apache/fory/util/StringUtils.java index 43cb809407..00331df645 100644 --- a/java/fory-core/src/main/java/org/apache/fory/util/StringUtils.java +++ b/java/fory-core/src/main/java/org/apache/fory/util/StringUtils.java @@ -26,7 +26,6 @@ import java.util.Map; import java.util.Random; import org.apache.fory.memory.NativeByteOrder; -import org.apache.fory.platform.UnsafeOps; public class StringUtils { // A long mask used to clear all-higher bits of char in a super-word way. @@ -278,40 +277,6 @@ public static String lowerCamelToLowerUnderscore(String lowerCamel) { return builder.toString(); } - public static boolean isLatin(char[] chars) { - return isLatin(chars, 0); - } - - public static boolean isLatin(char[] chars, int start) { - if (start > chars.length) { - return false; - } - int byteOffset = start << 1; - int numChars = chars.length; - int vectorizedLen = numChars >> 2; - int vectorizedChars = vectorizedLen << 2; - int endOffset = UnsafeOps.CHAR_ARRAY_OFFSET + (vectorizedChars << 1); - boolean isLatin = true; - for (int offset = UnsafeOps.CHAR_ARRAY_OFFSET + byteOffset; offset < endOffset; offset += 8) { - // check 4 chars in a vectorized way, 4 times faster than scalar check loop. - // See benchmark in CompressStringSuite.latinSuperWordCheck. - long multiChars = UnsafeOps.getLong(chars, offset); - if ((multiChars & MULTI_CHARS_NON_LATIN_MASK) != 0) { - isLatin = false; - break; - } - } - if (isLatin) { - for (int i = vectorizedChars; i < numChars; i++) { - if (chars[i] > 0xFF) { - isLatin = false; - break; - } - } - } - return isLatin; - } - /** * Split a string from the right side, similar to Python's rsplit. * diff --git a/java/fory-core/src/main/java25/org/apache/fory/serializer/PlatformStringUtils.java b/java/fory-core/src/main/java25/org/apache/fory/serializer/PlatformStringUtils.java new file mode 100644 index 0000000000..e5ac8063cb --- /dev/null +++ b/java/fory-core/src/main/java25/org/apache/fory/serializer/PlatformStringUtils.java @@ -0,0 +1,139 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.serializer; + +import java.lang.invoke.MethodHandles; +import java.lang.invoke.VarHandle; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; +import org.apache.fory.memory.MemoryBuffer; +import org.apache.fory.memory.NativeByteOrder; +import org.apache.fory.platform.AndroidSupport; +import org.apache.fory.platform.GraalvmSupport; +import org.apache.fory.platform.internal._JDKAccess; + +/** JDK25 string internals used by {@link StringSerializer}. */ +final class PlatformStringUtils { + static final boolean JDK_STRING_FIELD_ACCESS = + !AndroidSupport.IS_ANDROID + && !GraalvmSupport.IN_GRAALVM_NATIVE_IMAGE + && _JDKAccess.JDK_STRING_FIELD_ACCESS; + static final boolean STRING_VALUE_FIELD_IS_CHARS = + JDK_STRING_FIELD_ACCESS && _JDKAccess.STRING_VALUE_FIELD_IS_CHARS; + static final boolean STRING_VALUE_FIELD_IS_BYTES = + JDK_STRING_FIELD_ACCESS && _JDKAccess.STRING_VALUE_FIELD_IS_BYTES; + static final boolean STRING_HAS_COUNT_OFFSET = + JDK_STRING_FIELD_ACCESS && _JDKAccess.STRING_HAS_COUNT_OFFSET; + + private static final byte LATIN1 = 0; + private static final byte UTF16 = 1; + private static final VarHandle BYTE_ARRAY_LONG = + MethodHandles.byteArrayViewVarHandle(long[].class, ByteOrder.nativeOrder()); + private static final VarHandle BYTE_ARRAY_CHAR = + MethodHandles.byteArrayViewVarHandle(char[].class, ByteOrder.nativeOrder()); + + private PlatformStringUtils() {} + + static Object getStringValue(String value) { + return _JDKAccess.getStringValue(value); + } + + static byte getStringCoder(String value) { + return _JDKAccess.getStringCoder(value); + } + + static int getStringOffset(String value) { + return _JDKAccess.getStringOffset(value); + } + + static int getStringCount(String value) { + return _JDKAccess.getStringCount(value); + } + + static String newCharsStringZeroCopy(char[] data) { + if (!JDK_STRING_FIELD_ACCESS) { + return new String(data); + } + return _JDKAccess.newCharsStringZeroCopy(data); + } + + static String newBytesStringZeroCopy(byte coder, byte[] data) { + if (!JDK_STRING_FIELD_ACCESS) { + return newBytesStringSlow(coder, data); + } + return _JDKAccess.newBytesStringZeroCopy(coder, data); + } + + private static String newBytesStringSlow(byte coder, byte[] data) { + if (coder == LATIN1) { + return new String(data, StandardCharsets.ISO_8859_1); + } else if (coder == UTF16) { + char[] chars = new char[data.length >> 1]; + for (int i = 0, j = 0; i < data.length; i += 2) { + chars[j++] = (char) ((data[i] & 0xff) | ((data[i + 1] & 0xff) << 8)); + } + return new String(chars); + } else { + return new String(data, StandardCharsets.UTF_8); + } + } + + static long getCharsLong(char[] chars, int charIndex) { + long c0 = chars[charIndex]; + long c1 = chars[charIndex + 1]; + long c2 = chars[charIndex + 2]; + long c3 = chars[charIndex + 3]; + if (NativeByteOrder.IS_LITTLE_ENDIAN) { + return c0 | (c1 << 16) | (c2 << 32) | (c3 << 48); + } else { + return (c0 << 48) | (c1 << 32) | (c2 << 16) | c3; + } + } + + static long getBytesLong(byte[] bytes, int byteIndex) { + return (long) BYTE_ARRAY_LONG.get(bytes, byteIndex); + } + + static char getBytesChar(byte[] bytes, int byteIndex) { + return (char) BYTE_ARRAY_CHAR.get(bytes, byteIndex); + } + + static void copyCharsToBytes( + char[] chars, int charOffset, byte[] target, int byteOffset, int numBytes) { + int charIndex = charOffset; + if (NativeByteOrder.IS_LITTLE_ENDIAN) { + for (int i = byteOffset, end = byteOffset + numBytes; i < end; i += 2) { + char c = chars[charIndex++]; + target[i] = (byte) c; + target[i + 1] = (byte) (c >>> 8); + } + } else { + for (int i = byteOffset, end = byteOffset + numBytes; i < end; i += 2) { + char c = chars[charIndex++]; + target[i] = (byte) (c >>> 8); + target[i + 1] = (byte) c; + } + } + } + + static void putBytes(MemoryBuffer buffer, int writerIndex, byte[] bytes, int numBytes) { + buffer.put(writerIndex, bytes, 0, numBytes); + } +} diff --git a/java/fory-core/src/main/java25/org/apache/fory/serializer/Serializers.java b/java/fory-core/src/main/java25/org/apache/fory/serializer/Serializers.java index 6e99a27848..627461be29 100644 --- a/java/fory-core/src/main/java25/org/apache/fory/serializer/Serializers.java +++ b/java/fory-core/src/main/java25/org/apache/fory/serializer/Serializers.java @@ -70,7 +70,6 @@ import org.apache.fory.serializer.scala.SingletonMapSerializer; import org.apache.fory.serializer.scala.SingletonObjectSerializer; import org.apache.fory.util.ExceptionUtils; -import org.apache.fory.util.StringUtils; /** Serialization utils and common serializers. */ @SuppressWarnings({"rawtypes", "unchecked"}) @@ -489,7 +488,7 @@ public void write(WriteContext writeContext, T value) { return; } char[] v = (char[]) rawValue; - if (StringUtils.isLatin(v)) { + if (StringEncodingUtils.isLatin(v)) { stringSerializer.writeCharsLatin1(buffer, v, value.length()); } else { stringSerializer.writeCharsUTF16(buffer, v, value.length()); diff --git a/java/fory-core/src/main/java25/org/apache/fory/serializer/SlicedStringUtil.java b/java/fory-core/src/main/java25/org/apache/fory/serializer/SlicedStringUtil.java deleted file mode 100644 index 924c16d1f6..0000000000 --- a/java/fory-core/src/main/java25/org/apache/fory/serializer/SlicedStringUtil.java +++ /dev/null @@ -1,285 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.fory.serializer; - -import org.apache.fory.memory.LittleEndian; -import org.apache.fory.memory.MemoryBuffer; -import org.apache.fory.memory.NativeByteOrder; -import org.apache.fory.platform.UnsafeOps; -import org.apache.fory.util.MathUtils; -import org.apache.fory.util.StringEncodingUtils; -import org.apache.fory.util.StringUtils; - -final class SlicedStringUtil { - private static final byte LATIN1 = 0; - private static final byte UTF16 = 1; - private static final byte UTF8 = 2; - - private SlicedStringUtil() {} - - static void writeCharsLatin1WithOffset( - StringSerializer serializer, MemoryBuffer buffer, char[] chars, int offset, int count) { - int writerIndex = buffer.writerIndex(); - long header = ((long) count << 2) | LATIN1; - buffer.ensure(writerIndex + 5 + count); - byte[] targetArray = buffer.getHeapMemory(); - if (targetArray != null) { - final int targetIndex = buffer._unsafeHeapWriterIndex(); - int arrIndex = targetIndex; - arrIndex += LittleEndian.putVarUint36Small(targetArray, arrIndex, header); - writerIndex += arrIndex - targetIndex; - for (int i = 0; i < count; i++) { - targetArray[arrIndex + i] = (byte) chars[offset + i]; - } - } else { - writerIndex += buffer._unsafePutVarUint36Small(writerIndex, header); - final byte[] tmpArray = serializer.getByteArray(count); - for (int i = 0; i < count; i++) { - tmpArray[i] = (byte) chars[offset + i]; - } - buffer.put(writerIndex, tmpArray, 0, count); - } - writerIndex += count; - buffer._unsafeWriterIndex(writerIndex); - } - - static void writeCharsUTF16WithOffset( - StringSerializer serializer, MemoryBuffer buffer, char[] chars, int offset, int count) { - int numBytes = MathUtils.doubleExact(count); - int writerIndex = buffer.writerIndex(); - long header = ((long) numBytes << 2) | UTF16; - buffer.ensure(writerIndex + 5 + numBytes); - final byte[] targetArray = buffer.getHeapMemory(); - if (targetArray != null) { - final int targetIndex = buffer._unsafeHeapWriterIndex(); - int arrIndex = targetIndex; - arrIndex += LittleEndian.putVarUint36Small(targetArray, arrIndex, header); - writerIndex += arrIndex - targetIndex + numBytes; - if (NativeByteOrder.IS_LITTLE_ENDIAN) { - writeCharsUTF16BEToHeap(chars, offset, arrIndex, numBytes, targetArray); - } else { - writeCharsUTF16BEToHeap(chars, offset, arrIndex, numBytes, targetArray); - } - } else { - writerIndex += buffer._unsafePutVarUint36Small(writerIndex, header); - if (NativeByteOrder.IS_LITTLE_ENDIAN) { - writerIndex = - offHeapWriteCharsUTF16WithOffset( - serializer, buffer, chars, offset, writerIndex, numBytes); - } else { - writerIndex = - offHeapWriteCharsUTF16BEWithOffset( - serializer, buffer, chars, offset, writerIndex, numBytes); - } - } - buffer._unsafeWriterIndex(writerIndex); - } - - static void writeCharsUTF8WithOffset( - StringSerializer serializer, MemoryBuffer buffer, char[] chars, int offset, int count) { - int estimateMaxBytes = count * 3; - int approxNumBytes = (int) (count * 1.5) + 1; - int writerIndex = buffer.writerIndex(); - buffer.ensure(writerIndex + 9 + estimateMaxBytes); - byte[] targetArray = buffer.getHeapMemory(); - if (targetArray != null) { - int targetIndex = buffer._unsafeHeapWriterIndex(); - int headerPos = targetIndex; - int arrIndex = targetIndex; - long header = ((long) approxNumBytes << 2) | UTF8; - int headerBytesWritten = LittleEndian.putVarUint36Small(targetArray, arrIndex, header); - arrIndex += headerBytesWritten; - writerIndex += headerBytesWritten; - targetIndex = - StringEncodingUtils.convertUTF16ToUTF8(chars, offset, count, targetArray, arrIndex); - byte stashedByte = targetArray[arrIndex]; - int written = targetIndex - arrIndex; - header = ((long) written << 2) | UTF8; - int diff = - LittleEndian.putVarUint36Small(targetArray, headerPos, header) - headerBytesWritten; - if (diff != 0) { - handleWriteCharsUTF8UnalignedHeaderBytes(targetArray, arrIndex, diff, written, stashedByte); - } - buffer._unsafeWriterIndex(writerIndex + written + diff); - } else { - final byte[] tmpArray = serializer.getByteArray(estimateMaxBytes); - int written = StringEncodingUtils.convertUTF16ToUTF8(chars, offset, count, tmpArray, 0); - long header = ((long) written << 2) | UTF8; - writerIndex += buffer._unsafePutVarUint36Small(writerIndex, header); - buffer.put(writerIndex, tmpArray, 0, written); - buffer._unsafeWriterIndex(writerIndex + written); - } - } - - static void writeCharsUTF8PerfOptimizedWithOffset( - StringSerializer serializer, MemoryBuffer buffer, char[] chars, int offset, int count) { - int estimateMaxBytes = count * 3; - int numBytes = MathUtils.doubleExact(count); - int writerIndex = buffer.writerIndex(); - long header = ((long) numBytes << 2) | UTF8; - buffer.ensure(writerIndex + 9 + estimateMaxBytes); - byte[] targetArray = buffer.getHeapMemory(); - if (targetArray != null) { - int targetIndex = buffer._unsafeHeapWriterIndex(); - int arrIndex = targetIndex; - arrIndex += LittleEndian.putVarUint36Small(targetArray, arrIndex, header); - writerIndex += arrIndex - targetIndex; - targetIndex = - StringEncodingUtils.convertUTF16ToUTF8(chars, offset, count, targetArray, arrIndex + 4); - int written = targetIndex - arrIndex - 4; - buffer._unsafePutInt32(writerIndex, written); - buffer._unsafeWriterIndex(writerIndex + 4 + written); - } else { - final byte[] tmpArray = serializer.getByteArray(estimateMaxBytes); - int written = StringEncodingUtils.convertUTF16ToUTF8(chars, offset, count, tmpArray, 0); - writerIndex += buffer._unsafePutVarUint36Small(writerIndex, header); - buffer._unsafePutInt32(writerIndex, written); - writerIndex += 4; - buffer.put(writerIndex, tmpArray, 0, written); - buffer._unsafeWriterIndex(writerIndex + written); - } - } - - static boolean isLatin(char[] chars, int offset, int count) { - int end = offset + count; - int vectorizedChars = count & ~3; - int vectorEnd = offset + vectorizedChars; - long byteOffset = UnsafeOps.CHAR_ARRAY_OFFSET + ((long) offset << 1); - long endOffset = UnsafeOps.CHAR_ARRAY_OFFSET + ((long) vectorEnd << 1); - for (long off = byteOffset; off < endOffset; off += 8) { - long multiChars = UnsafeOps.getLong(chars, off); - if ((multiChars & StringUtils.MULTI_CHARS_NON_LATIN_MASK) != 0) { - return false; - } - } - for (int i = vectorEnd; i < end; i++) { - if (chars[i] > 0xFF) { - return false; - } - } - return true; - } - - static byte bestCoder(char[] chars, int offset, int count) { - int sampleNum = Math.min(64, count); - int vectorizedLen = sampleNum >> 2; - int vectorizedChars = vectorizedLen << 2; - long byteOffset = UnsafeOps.CHAR_ARRAY_OFFSET + ((long) offset << 1); - long endOffset = byteOffset + ((long) vectorizedChars << 1); - int asciiCount = 0; - int latin1Count = 0; - int charOffset = offset; - for (long off = byteOffset; off < endOffset; off += 8, charOffset += 4) { - long multiChars = UnsafeOps.getLong(chars, off); - if ((multiChars & StringUtils.MULTI_CHARS_NON_ASCII_MASK) == 0) { - latin1Count += 4; - asciiCount += 4; - } else if ((multiChars & StringUtils.MULTI_CHARS_NON_LATIN_MASK) == 0) { - latin1Count += 4; - for (int i = 0; i < 4; ++i) { - if (chars[charOffset + i] < 0x80) { - asciiCount++; - } - } - } else { - for (int i = 0; i < 4; ++i) { - char c = chars[charOffset + i]; - if (c < 0x80) { - latin1Count++; - asciiCount++; - } else if (c <= 0xFF) { - latin1Count++; - } - } - } - } - - for (int i = vectorizedChars; i < sampleNum; i++) { - char c = chars[offset + i]; - if (c < 0x80) { - latin1Count++; - asciiCount++; - } else if (c <= 0xFF) { - latin1Count++; - } - } - - if (latin1Count == count || (latin1Count == sampleNum && isLatin(chars, offset, count))) { - return LATIN1; - } else if (asciiCount >= sampleNum * 0.5) { - return UTF8; - } else { - return UTF16; - } - } - - private static void handleWriteCharsUTF8UnalignedHeaderBytes( - byte[] targetArray, int arrIndex, int diff, int written, byte stashed) { - if (diff == 1) { - System.arraycopy(targetArray, arrIndex + 1, targetArray, arrIndex + 2, written - 1); - targetArray[arrIndex + 1] = stashed; - } else { - System.arraycopy(targetArray, arrIndex, targetArray, arrIndex - 1, written); - } - } - - private static void writeCharsUTF16BEToHeap( - char[] chars, int offset, int arrIndex, int numBytes, byte[] targetArray) { - int charIndex = offset; - for (int i = arrIndex, end = i + numBytes; i < end; i += 2) { - char c = chars[charIndex++]; - targetArray[i] = (byte) c; - targetArray[i + 1] = (byte) (c >>> 8); - } - } - - private static int offHeapWriteCharsUTF16WithOffset( - StringSerializer serializer, - MemoryBuffer buffer, - char[] chars, - int offset, - int writerIndex, - int numBytes) { - byte[] tmpArray = serializer.getByteArray(numBytes); - writeCharsUTF16BEToHeap(chars, offset, 0, numBytes, tmpArray); - buffer.put(writerIndex, tmpArray, 0, numBytes); - writerIndex += numBytes; - return writerIndex; - } - - private static int offHeapWriteCharsUTF16BEWithOffset( - StringSerializer serializer, - MemoryBuffer buffer, - char[] chars, - int offset, - int writerIndex, - int numBytes) { - byte[] tmpArray = serializer.getByteArray(numBytes); - int charIndex = offset; - for (int i = 0; i < numBytes; i += 2) { - char c = chars[charIndex++]; - tmpArray[i] = (byte) c; - tmpArray[i + 1] = (byte) (c >>> 8); - } - buffer.put(writerIndex, tmpArray, 0, numBytes); - writerIndex += numBytes; - return writerIndex; - } -} diff --git a/java/fory-core/src/main/java25/org/apache/fory/serializer/StringSerializer.java b/java/fory-core/src/main/java25/org/apache/fory/serializer/StringSerializer.java deleted file mode 100644 index 92cbb62154..0000000000 --- a/java/fory-core/src/main/java25/org/apache/fory/serializer/StringSerializer.java +++ /dev/null @@ -1,1171 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.fory.serializer; - -import static org.apache.fory.type.TypeUtils.STRING_TYPE; -import static org.apache.fory.util.StringUtils.MULTI_CHARS_NON_ASCII_MASK; -import static org.apache.fory.util.StringUtils.MULTI_CHARS_NON_LATIN_MASK; - -import java.nio.charset.StandardCharsets; -import java.util.Arrays; -import org.apache.fory.annotation.CodegenInvoke; -import org.apache.fory.codegen.Expression; -import org.apache.fory.codegen.Expression.Invoke; -import org.apache.fory.codegen.Expression.StaticInvoke; -import org.apache.fory.config.Config; -import org.apache.fory.context.ReadContext; -import org.apache.fory.context.WriteContext; -import org.apache.fory.memory.LittleEndian; -import org.apache.fory.memory.MemoryBuffer; -import org.apache.fory.memory.NativeByteOrder; -import org.apache.fory.platform.AndroidSupport; -import org.apache.fory.platform.GraalvmSupport; -import org.apache.fory.platform.UnsafeOps; -import org.apache.fory.platform.internal._JDKAccess; -import org.apache.fory.util.MathUtils; -import org.apache.fory.util.Preconditions; -import org.apache.fory.util.StringEncodingUtils; -import org.apache.fory.util.StringUtils; - -/** - * String serializer based on method handles and byte-array accessors for speed. - * - *

Note that string operations is very common in serialization, and jvm inline and branch - * elimination is not reliable even in c2 compiler, so we try to inline and avoid checks as we can - * manually. - */ -@SuppressWarnings("unchecked") -public final class StringSerializer extends ImmutableSerializer { - private static final boolean STRING_VALUE_FIELD_IS_CHARS; - private static final boolean STRING_VALUE_FIELD_IS_BYTES; - - private static final byte LATIN1 = 0; - private static final byte UTF16 = 1; - private static final byte UTF8 = 2; - private static final int DEFAULT_BUFFER_SIZE = 1024; - - private static final boolean STRING_HAS_COUNT_OFFSET; - - static { - if (!jdkInternalFieldAccess()) { - STRING_VALUE_FIELD_IS_CHARS = false; - STRING_VALUE_FIELD_IS_BYTES = false; - STRING_HAS_COUNT_OFFSET = false; - } else { - STRING_VALUE_FIELD_IS_CHARS = _JDKAccess.STRING_VALUE_FIELD_IS_CHARS; - STRING_VALUE_FIELD_IS_BYTES = _JDKAccess.STRING_VALUE_FIELD_IS_BYTES; - STRING_HAS_COUNT_OFFSET = _JDKAccess.STRING_HAS_COUNT_OFFSET; - } - } - - private static boolean jdkInternalFieldAccess() { - // Native-image runtime string layout is not a HotSpot private-field contract; use public - // string copies there even when the image build can see JDK private fields. - return !AndroidSupport.IS_ANDROID - && !GraalvmSupport.IN_GRAALVM_NATIVE_IMAGE - && _JDKAccess.JDK_STRING_FIELD_ACCESS; - } - - private final boolean compressString; - private final boolean writeNumUtf16BytesForUtf8Encoding; - private final boolean xlang; - - // set default length to 0, since char array and bytes array won't be used at the same time. - private static final byte[] EMPTY_BYTES_STUB = new byte[0]; - private static final char[] EMPTY_CHARS_STUB = new char[0]; - private byte[] byteArray = EMPTY_BYTES_STUB; - private int smoothByteArrayLength = DEFAULT_BUFFER_SIZE; - private char[] charArray = EMPTY_CHARS_STUB; - private int smoothCharArrayLength = DEFAULT_BUFFER_SIZE; - private byte[] byteArray2 = EMPTY_BYTES_STUB; - - public StringSerializer(Config config) { - super(config, String.class, config.trackingRef() && !config.isStringRefIgnored()); - compressString = config.compressString(); - xlang = config.isXlang(); - if (xlang) { - Preconditions.checkArgument(compressString, "compress string muse be enabled for xlang mode"); - } - writeNumUtf16BytesForUtf8Encoding = config.writeNumUtf16BytesForUtf8Encoding(); - } - - @Override - public void write(WriteContext writeContext, String value) { - writeString(writeContext.getBuffer(), value); - } - - @Override - public String read(ReadContext readContext) { - return readString(readContext.getBuffer()); - } - - public static Expression writeStringExpr( - Expression strSerializer, Expression buffer, Expression str, boolean compressString) { - if (!jdkInternalFieldAccess()) { - return new Invoke(strSerializer, "writeString", buffer, str); - } - if (STRING_VALUE_FIELD_IS_BYTES) { - if (compressString) { - return new Invoke(strSerializer, "writeCompressedBytesString", buffer, str); - } else { - return new StaticInvoke(StringSerializer.class, "writeBytesString", buffer, str); - } - } else { - if (!STRING_VALUE_FIELD_IS_CHARS) { - throw new UnsupportedOperationException(); - } - if (STRING_HAS_COUNT_OFFSET) { - if (compressString) { - return new Invoke(strSerializer, "writeCompressedCharsStringWithOffset", buffer, str); - } else { - return new Invoke(strSerializer, "writeCharsStringWithOffset", buffer, str); - } - } else { - if (compressString) { - return new Invoke(strSerializer, "writeCompressedCharsString", buffer, str); - } else { - return new Invoke(strSerializer, "writeCharsString", buffer, str); - } - } - } - } - - public static Expression readStringExpr( - Expression strSerializer, Expression buffer, boolean compressString) { - if (!jdkInternalFieldAccess()) { - return new Invoke(strSerializer, "readString", STRING_TYPE, buffer); - } - if (STRING_VALUE_FIELD_IS_BYTES) { - if (compressString) { - return new Invoke(strSerializer, "readCompressedBytesString", STRING_TYPE, buffer); - } else { - return new Invoke(strSerializer, "readBytesString", STRING_TYPE, buffer); - } - } else { - if (!STRING_VALUE_FIELD_IS_CHARS) { - throw new UnsupportedOperationException(); - } - if (compressString) { - return new Invoke(strSerializer, "readCompressedCharsString", STRING_TYPE, buffer); - } else { - return new Invoke(strSerializer, "readCharsString", STRING_TYPE, buffer); - } - } - } - - @CodegenInvoke - public String readBytesString(MemoryBuffer buffer) { - long header = buffer.readVarUint36Small(); - byte coder = (byte) (header & 0b11); - int numBytes = (int) (header >>> 2); - byte[] bytes; - if (!NativeByteOrder.IS_LITTLE_ENDIAN && coder == UTF16) { - bytes = readBytesUTF16BE(buffer, numBytes); - } else { - bytes = readBytesUnCompressedUTF16(buffer, numBytes); - } - if (coder != UTF8) { - return newBytesStringZeroCopy(coder, bytes); - } else { - return new String(bytes, 0, numBytes, StandardCharsets.UTF_8); - } - } - - @CodegenInvoke - public String readCharsString(MemoryBuffer buffer) { - long header = buffer.readVarUint36Small(); - byte coder = (byte) (header & 0b11); - int numBytes = (int) (header >>> 2); - char[] chars; - if (coder == LATIN1) { - chars = readCharsLatin1(buffer, numBytes); - } else if (coder == UTF16) { - chars = readCharsUTF16(buffer, numBytes); - } else { - throw new RuntimeException("Unknown coder type " + coder); - } - return newCharsStringZeroCopy(chars); - } - - @CodegenInvoke - public String readCompressedBytesString(MemoryBuffer buffer) { - long header = buffer.readVarUint36Small(); - byte coder = (byte) (header & 0b11); - int numBytes = (int) (header >>> 2); - if (coder == UTF8) { - byte[] data; - if (writeNumUtf16BytesForUtf8Encoding) { - data = readBytesUTF8PerfOptimized(buffer, numBytes); - } else { - if (xlang) { - return readBytesUTF8ForXlang(buffer, numBytes); - } - data = readBytesUTF8(buffer, numBytes); - } - return newBytesStringZeroCopy(UTF16, data); - } else if (coder == LATIN1) { - return newBytesStringZeroCopy(coder, readBytesUnCompressedUTF16(buffer, numBytes)); - } else if (coder == UTF16) { - byte[] bytes; - if (NativeByteOrder.IS_LITTLE_ENDIAN) { - bytes = readBytesUnCompressedUTF16(buffer, numBytes); - } else { - bytes = readBytesUTF16BE(buffer, numBytes); - } - return newBytesStringZeroCopy(coder, bytes); - } else { - throw new RuntimeException("Unknown coder type " + coder); - } - } - - // the utf8 data may can be encoded with latin1, so the read need to check whether it can be - // encoded by latin1, if true, the coder should be latin1 instead of utf16 - String readBytesUTF8ForXlang(MemoryBuffer buffer, int numBytes) { - buffer.checkReadableBytes(numBytes); - byte[] srcArray = buffer.getHeapMemory(); - - if (srcArray != null) { - int srcIndex = buffer._unsafeHeapReaderIndex(); - - // Fast path: vectorized ASCII check (8 bytes at a time) - if (StringEncodingUtils.isUTF8WithinAscii(srcArray, srcIndex, numBytes)) { - byte[] result = new byte[numBytes]; - System.arraycopy(srcArray, srcIndex, result, 0, numBytes); - buffer._increaseReaderIndexUnsafe(numBytes); - return newBytesStringZeroCopy(LATIN1, result); - } - - // Two-pass approach: scan first, then convert - boolean isLatin1 = StringEncodingUtils.isUTF8WithinLatin1(srcArray, srcIndex, numBytes); - buffer._increaseReaderIndexUnsafe(numBytes); - - if (isLatin1) { - byte[] latin1Buffer = getByteArray(numBytes); - int latin1Len = - StringEncodingUtils.convertUTF8ToLatin1(srcArray, srcIndex, numBytes, latin1Buffer); - return newBytesStringZeroCopy(LATIN1, Arrays.copyOf(latin1Buffer, latin1Len)); - } else { - byte[] utf16Buffer = getByteArray(numBytes << 1); - int utf16Len = - StringEncodingUtils.convertUTF8ToUTF16(srcArray, srcIndex, numBytes, utf16Buffer); - return newBytesStringZeroCopy(UTF16, Arrays.copyOf(utf16Buffer, utf16Len)); - } - } else { - // Off-heap path - byte[] srcBytes = getByteArray2(numBytes); - buffer.readBytes(srcBytes, 0, numBytes); - - // Fast path: vectorized ASCII check - if (StringEncodingUtils.isUTF8WithinAscii(srcBytes, 0, numBytes)) { - // Must copy to exact size since srcBytes is a reusable buffer - return newBytesStringZeroCopy(LATIN1, Arrays.copyOf(srcBytes, numBytes)); - } - - // Two-pass approach: scan first, then convert - boolean isLatin1 = StringEncodingUtils.isUTF8WithinLatin1(srcBytes, 0, numBytes); - - if (isLatin1) { - byte[] latin1Buffer = getByteArray(numBytes); - int latin1Len = - StringEncodingUtils.convertUTF8ToLatin1(srcBytes, 0, numBytes, latin1Buffer); - return newBytesStringZeroCopy(LATIN1, Arrays.copyOf(latin1Buffer, latin1Len)); - } else { - byte[] utf16Buffer = getByteArray(numBytes << 1); - int utf16Len = StringEncodingUtils.convertUTF8ToUTF16(srcBytes, 0, numBytes, utf16Buffer); - return newBytesStringZeroCopy(UTF16, Arrays.copyOf(utf16Buffer, utf16Len)); - } - } - } - - @CodegenInvoke - public String readCompressedCharsString(MemoryBuffer buffer) { - long header = buffer.readVarUint36Small(); - byte coder = (byte) (header & 0b11); - int numBytes = (int) (header >>> 2); - char[] chars; - if (coder == LATIN1) { - chars = readCharsLatin1(buffer, numBytes); - } else if (coder == UTF8) { - return writeNumUtf16BytesForUtf8Encoding - ? readCharsUTF8PerfOptimized(buffer, numBytes) - : readCharsUTF8(buffer, numBytes); - } else if (coder == UTF16) { - chars = readCharsUTF16(buffer, numBytes); - } else { - throw new RuntimeException("Unknown coder type " + coder); - } - return newCharsStringZeroCopy(chars); - } - - // Invoked by fory JIT - public void writeString(MemoryBuffer buffer, String value) { - if (!jdkInternalFieldAccess()) { - writeStringSlow(buffer, value); - return; - } - if (STRING_VALUE_FIELD_IS_BYTES) { - if (compressString) { - writeCompressedBytesString(buffer, value); - } else { - writeBytesString(buffer, value); - } - } else { - writeJava8String(buffer, value); - } - } - - private void writeJava8String(MemoryBuffer buffer, String value) { - assert STRING_VALUE_FIELD_IS_CHARS; - if (STRING_HAS_COUNT_OFFSET) { - if (compressString) { - writeCompressedCharsStringWithOffset(buffer, value); - } else { - writeCharsStringWithOffset(buffer, value); - } - } else { - if (compressString) { - writeCompressedCharsString(buffer, value); - } else { - writeCharsString(buffer, value); - } - } - } - - // Invoked by fory JIT - public String readString(MemoryBuffer buffer) { - if (!jdkInternalFieldAccess()) { - return readStringSlow(buffer); - } - if (STRING_VALUE_FIELD_IS_BYTES) { - if (compressString) { - return readCompressedBytesString(buffer); - } else { - return readBytesString(buffer); - } - } else { - assert STRING_VALUE_FIELD_IS_CHARS; - if (compressString) { - return readCompressedCharsString(buffer); - } else { - return readCharsString(buffer); - } - } - } - - private void writeStringSlow(MemoryBuffer buffer, String value) { - char[] chars = value.toCharArray(); - if (isLatin(chars)) { - writeCharsLatin1(buffer, chars, chars.length); - return; - } - if (compressString) { - byte[] utf8Bytes = value.getBytes(StandardCharsets.UTF_8); - int utf16Bytes = chars.length << 1; - if (utf8Bytes.length < utf16Bytes) { - writeStringUtf8Slow(buffer, utf8Bytes, utf16Bytes); - return; - } - } - writeCharsUTF16(buffer, chars, chars.length); - } - - private String readStringSlow(MemoryBuffer buffer) { - long header = buffer.readVarUint36Small(); - byte coder = (byte) (header & 0b11); - int numBytes = (int) (header >>> 2); - if (coder == LATIN1) { - return new String(readBytesUnCompressedUTF16(buffer, numBytes), StandardCharsets.ISO_8859_1); - } else if (coder == UTF16) { - return new String(readCharsUTF16(buffer, numBytes)); - } else if (coder == UTF8) { - int utf8Bytes = writeNumUtf16BytesForUtf8Encoding ? buffer.readInt32() : numBytes; - return new String(buffer.readBytes(utf8Bytes), StandardCharsets.UTF_8); - } else { - throw new RuntimeException("Unknown coder type " + coder); - } - } - - private void writeStringUtf8Slow(MemoryBuffer buffer, byte[] utf8Bytes, int utf16Bytes) { - int headerLength = writeNumUtf16BytesForUtf8Encoding ? utf16Bytes : utf8Bytes.length; - writeVarUint36Small(buffer, ((long) headerLength << 2) | UTF8); - if (writeNumUtf16BytesForUtf8Encoding) { - buffer.writeInt32(utf8Bytes.length); - } - buffer.writeBytes(utf8Bytes); - } - - private static void writeVarUint36Small(MemoryBuffer buffer, long value) { - int writerIndex = buffer.writerIndex(); - buffer.ensure(writerIndex + 9); - writerIndex += buffer._unsafePutVarUint36Small(writerIndex, value); - buffer._unsafeWriterIndex(writerIndex); - } - - private static boolean isLatin(char[] chars) { - for (char c : chars) { - if (c > 0xFF) { - return false; - } - } - return true; - } - - private static Object getStringValue(String value) { - return _JDKAccess.getStringValue(value); - } - - private static byte getStringCoder(String value) { - return _JDKAccess.getStringCoder(value); - } - - private static int getStringOffset(String value) { - return _JDKAccess.getStringOffset(value); - } - - private static int getStringCount(String value) { - return _JDKAccess.getStringCount(value); - } - - @CodegenInvoke - public void writeCompressedBytesString(MemoryBuffer buffer, String value) { - final byte[] bytes = (byte[]) getStringValue(value); - final byte coder = getStringCoder(value); - if (coder == LATIN1 || bestCoder(bytes) == UTF16) { - writeBytesString(buffer, coder, bytes); - } else { - if (writeNumUtf16BytesForUtf8Encoding) { - writeBytesUTF8PerfOptimized(buffer, bytes); - } else { - writeBytesUTF8(buffer, bytes); - } - } - } - - @CodegenInvoke - public void writeCompressedCharsString(MemoryBuffer buffer, String value) { - final char[] chars = (char[]) getStringValue(value); - final byte coder = bestCoder(chars); - if (coder == LATIN1) { - writeCharsLatin1(buffer, chars, chars.length); - } else if (coder == UTF8) { - if (writeNumUtf16BytesForUtf8Encoding) { - writeCharsUTF8PerfOptimized(buffer, chars); - } else { - writeCharsUTF8(buffer, chars); - } - } else { - writeCharsUTF16(buffer, chars, chars.length); - } - } - - @CodegenInvoke - public void writeCompressedCharsStringWithOffset(MemoryBuffer buffer, String value) { - final char[] chars = (char[]) getStringValue(value); - final int offset = getStringOffset(value); - final int count = getStringCount(value); - final byte coder = SlicedStringUtil.bestCoder(chars, offset, count); - if (coder == LATIN1) { - SlicedStringUtil.writeCharsLatin1WithOffset(this, buffer, chars, offset, count); - } else if (coder == UTF8) { - if (writeNumUtf16BytesForUtf8Encoding) { - SlicedStringUtil.writeCharsUTF8PerfOptimizedWithOffset(this, buffer, chars, offset, count); - } else { - SlicedStringUtil.writeCharsUTF8WithOffset(this, buffer, chars, offset, count); - } - } else { - SlicedStringUtil.writeCharsUTF16WithOffset(this, buffer, chars, offset, count); - } - } - - @CodegenInvoke - public static void writeBytesString(MemoryBuffer buffer, String value) { - byte[] bytes = (byte[]) getStringValue(value); - byte coder = getStringCoder(value); - writeBytesString(buffer, coder, bytes); - } - - public static void writeBytesString(MemoryBuffer buffer, byte coder, byte[] bytes) { - if (!NativeByteOrder.IS_LITTLE_ENDIAN && coder == UTF16) { - writeBytesStringUTF16BE(buffer, bytes); - return; - } - int bytesLen = bytes.length; - long header = ((long) bytesLen << 2) | coder; - int writerIndex = buffer.writerIndex(); - // The `ensure` ensure next operations are safe without bound checks, - // and inner heap buffer doesn't change. - buffer.ensure(writerIndex + 9 + bytesLen); // 1 byte coder + varint max 8 bytes - final byte[] targetArray = buffer.getHeapMemory(); - if (targetArray != null) { - // Some JDK11 Unsafe.copyMemory will `copyMemoryChecks`, and - // jvm doesn't eliminate well in some jdk. - final int targetIndex = buffer._unsafeHeapWriterIndex(); - int arrIndex = targetIndex; - arrIndex += LittleEndian.putVarUint36Small(targetArray, arrIndex, header); - writerIndex += arrIndex - targetIndex; - System.arraycopy(bytes, 0, targetArray, arrIndex, bytesLen); - } else { - final int headerBytes = buffer._unsafePutVarUint36Small(writerIndex, header); - writerIndex += headerBytes; - buffer.put(writerIndex, bytes, 0, bytesLen); - } - writerIndex += bytesLen; - buffer._unsafeWriterIndex(writerIndex); - } - - @CodegenInvoke - public void writeCharsString(MemoryBuffer buffer, String value) { - final char[] chars = (char[]) getStringValue(value); - if (StringUtils.isLatin(chars)) { - writeCharsLatin1(buffer, chars, chars.length); - } else { - writeCharsUTF16(buffer, chars, chars.length); - } - } - - @CodegenInvoke - public void writeCharsStringWithOffset(MemoryBuffer buffer, String value) { - final char[] chars = (char[]) getStringValue(value); - final int offset = getStringOffset(value); - final int count = getStringCount(value); - if (SlicedStringUtil.isLatin(chars, offset, count)) { - SlicedStringUtil.writeCharsLatin1WithOffset(this, buffer, chars, offset, count); - } else { - SlicedStringUtil.writeCharsUTF16WithOffset(this, buffer, chars, offset, count); - } - } - - public char[] readCharsLatin1(MemoryBuffer buffer, int numBytes) { - buffer.checkReadableBytes(numBytes); - byte[] srcArray = buffer.getHeapMemory(); - char[] chars = new char[numBytes]; - if (srcArray != null) { - int srcIndex = buffer._unsafeHeapReaderIndex(); - for (int i = 0; i < numBytes; i++) { - chars[i] = (char) (srcArray[srcIndex++] & 0xff); - } - buffer._increaseReaderIndexUnsafe(numBytes); - } else { - byte[] tmpArray = getByteArray(numBytes); - buffer.readBytes(tmpArray, 0, numBytes); - for (int i = 0; i < numBytes; i++) { - chars[i] = (char) (tmpArray[i] & 0xff); - } - } - return chars; - } - - public byte[] readBytesUTF8(MemoryBuffer buffer, int numBytes) { - byte[] tmpArray = getByteArray(numBytes << 1); - buffer.checkReadableBytes(numBytes); - int utf16NumBytes; - byte[] srcArray = buffer.getHeapMemory(); - if (srcArray != null) { - int srcIndex = buffer._unsafeHeapReaderIndex(); - utf16NumBytes = - StringEncodingUtils.convertUTF8ToUTF16(srcArray, srcIndex, numBytes, tmpArray); - buffer._increaseReaderIndexUnsafe(numBytes); - } else { - byte[] byteArray2 = getByteArray2(numBytes); - buffer.readBytes(byteArray2, 0, numBytes); - utf16NumBytes = StringEncodingUtils.convertUTF8ToUTF16(byteArray2, 0, numBytes, tmpArray); - } - return Arrays.copyOf(tmpArray, utf16NumBytes); - } - - private byte[] readBytesUTF8PerfOptimized(MemoryBuffer buffer, int numBytes) { - int udf8Bytes = buffer.readInt32(); - byte[] bytes = new byte[numBytes]; - // noinspection Duplicates - buffer.checkReadableBytes(udf8Bytes); - byte[] srcArray = buffer.getHeapMemory(); - if (srcArray != null) { - int srcIndex = buffer._unsafeHeapReaderIndex(); - int readLen = StringEncodingUtils.convertUTF8ToUTF16(srcArray, srcIndex, udf8Bytes, bytes); - assert readLen == numBytes : "Decode UTF8 to UTF16 failed"; - buffer._increaseReaderIndexUnsafe(udf8Bytes); - } else { - byte[] tmpArray = getByteArray(udf8Bytes); - buffer.readBytes(tmpArray, 0, udf8Bytes); - int readLen = StringEncodingUtils.convertUTF8ToUTF16(tmpArray, 0, udf8Bytes, bytes); - assert readLen == numBytes : "Decode UTF8 to UTF16 failed"; - } - return bytes; - } - - public byte[] readBytesUnCompressedUTF16(MemoryBuffer buffer, int numBytes) { - buffer.checkReadableBytes(numBytes); - byte[] bytes; - byte[] heapMemory = buffer.getHeapMemory(); - if (heapMemory != null) { - final int arrIndex = buffer._unsafeHeapReaderIndex(); - buffer.increaseReaderIndex(numBytes); - bytes = new byte[numBytes]; - System.arraycopy(heapMemory, arrIndex, bytes, 0, numBytes); - } else { - bytes = buffer.readBytes(numBytes); - } - return bytes; - } - - public char[] readCharsUTF16(MemoryBuffer buffer, int numBytes) { - if (NativeByteOrder.IS_LITTLE_ENDIAN) { - char[] chars = new char[numBytes >> 1]; - // FIXME JDK11 utf16 string uses little-endian order. - buffer.readChars(chars, numBytes >> 1); - return chars; - } else { - return readCharsUTF16BE(buffer, numBytes); - } - } - - public String readCharsUTF8(MemoryBuffer buffer, int numBytes) { - char[] chars = getCharArray(numBytes); - int charsLen; - buffer.checkReadableBytes(numBytes); - byte[] srcArray = buffer.getHeapMemory(); - if (srcArray != null) { - int srcIndex = buffer._unsafeHeapReaderIndex(); - charsLen = StringEncodingUtils.convertUTF8ToUTF16(srcArray, srcIndex, numBytes, chars); - buffer._increaseReaderIndexUnsafe(numBytes); - } else { - byte[] tmpArray = getByteArray(numBytes); - buffer.readBytes(tmpArray, 0, numBytes); - charsLen = StringEncodingUtils.convertUTF8ToUTF16(tmpArray, 0, numBytes, chars); - } - return new String(chars, 0, charsLen); - } - - public String readCharsUTF8PerfOptimized(MemoryBuffer buffer, int numBytes) { - int udf16Chars = numBytes >> 1; - int udf8Bytes = buffer.readInt32(); - char[] chars = new char[udf16Chars]; - // noinspection Duplicates - buffer.checkReadableBytes(udf8Bytes); - byte[] srcArray = buffer.getHeapMemory(); - if (srcArray != null) { - int srcIndex = buffer._unsafeHeapReaderIndex(); - int readLen = StringEncodingUtils.convertUTF8ToUTF16(srcArray, srcIndex, udf8Bytes, chars); - assert readLen == udf16Chars : "Decode UTF8 to UTF16 failed"; - buffer._increaseReaderIndexUnsafe(udf8Bytes); - } else { - byte[] tmpArray = getByteArray(udf8Bytes); - buffer.readBytes(tmpArray, 0, udf8Bytes); - int readLen = StringEncodingUtils.convertUTF8ToUTF16(tmpArray, 0, udf8Bytes, chars); - assert readLen == udf16Chars : "Decode UTF8 to UTF16 failed"; - } - return newCharsStringZeroCopy(chars); - } - - public void writeCharsLatin1(MemoryBuffer buffer, char[] chars, int numBytes) { - int writerIndex = buffer.writerIndex(); - long header = ((long) numBytes << 2) | LATIN1; - buffer.ensure(writerIndex + 5 + numBytes); - byte[] targetArray = buffer.getHeapMemory(); - if (targetArray != null) { - final int targetIndex = buffer._unsafeHeapWriterIndex(); - int arrIndex = targetIndex; - arrIndex += LittleEndian.putVarUint36Small(targetArray, arrIndex, header); - writerIndex += arrIndex - targetIndex; - for (int i = 0; i < numBytes; i++) { - targetArray[arrIndex + i] = (byte) chars[i]; - } - } else { - writerIndex += buffer._unsafePutVarUint36Small(writerIndex, header); - final byte[] tmpArray = getByteArray(numBytes); - for (int i = 0; i < numBytes; i++) { - tmpArray[i] = (byte) chars[i]; - } - buffer.put(writerIndex, tmpArray, 0, numBytes); - } - writerIndex += numBytes; - buffer._unsafeWriterIndex(writerIndex); - } - - public void writeCharsUTF16(MemoryBuffer buffer, char[] chars, int numChars) { - int numBytes = MathUtils.doubleExact(numChars); - int writerIndex = buffer.writerIndex(); - long header = ((long) numBytes << 2) | UTF16; - buffer.ensure(writerIndex + 5 + numBytes); - final byte[] targetArray = buffer.getHeapMemory(); - if (targetArray != null) { - final int targetIndex = buffer._unsafeHeapWriterIndex(); - int arrIndex = targetIndex; - arrIndex += LittleEndian.putVarUint36Small(targetArray, arrIndex, header); - writerIndex += arrIndex - targetIndex + numBytes; - if (NativeByteOrder.IS_LITTLE_ENDIAN) { - if (AndroidSupport.IS_ANDROID) { - writeCharsUTF16ToHeapSlow(chars, arrIndex, numBytes, targetArray); - } else { - writeCharsUTF16ToHeapSlow(chars, arrIndex, numBytes, targetArray); - } - } else { - writeCharsUTF16BEToHeap(chars, arrIndex, numBytes, targetArray); - } - } else { - writerIndex += buffer._unsafePutVarUint36Small(writerIndex, header); - if (NativeByteOrder.IS_LITTLE_ENDIAN) { - writerIndex = offHeapWriteCharsUTF16(buffer, chars, writerIndex, numBytes); - } else { - writerIndex = offHeapWriteCharsUTF16BE(buffer, chars, writerIndex, numBytes); - } - } - buffer._unsafeWriterIndex(writerIndex); - } - - public void writeCharsUTF8(MemoryBuffer buffer, char[] chars) { - int estimateMaxBytes = chars.length * 3; - // num bytes of utf8 should be smaller than utf16, otherwise we should - // utf16 instead. - // We can't use length in header since we don't know num chars in go/c++ - int approxNumBytes = (int) (chars.length * 1.5) + 1; - int writerIndex = buffer.writerIndex(); - // 9 for max bytes of header - buffer.ensure(writerIndex + 9 + estimateMaxBytes); - byte[] targetArray = buffer.getHeapMemory(); - if (targetArray != null) { - // noinspection Duplicates - int targetIndex = buffer._unsafeHeapWriterIndex(); - // keep this index in case actual num utf8 bytes need different bytes for header - int headerPos = targetIndex; - int arrIndex = targetIndex; - long header = ((long) approxNumBytes << 2) | UTF8; - int headerBytesWritten = LittleEndian.putVarUint36Small(targetArray, arrIndex, header); - arrIndex += headerBytesWritten; - writerIndex += headerBytesWritten; - // noinspection Duplicates - targetIndex = StringEncodingUtils.convertUTF16ToUTF8(chars, targetArray, arrIndex); - byte stashedByte = targetArray[arrIndex]; - int written = targetIndex - arrIndex; - header = ((long) written << 2) | UTF8; - int diff = - LittleEndian.putVarUint36Small(targetArray, headerPos, header) - headerBytesWritten; - if (diff != 0) { - handleWriteCharsUTF8UnalignedHeaderBytes(targetArray, arrIndex, diff, written, stashedByte); - } - buffer._unsafeWriterIndex(writerIndex + written + diff); - } else { - // noinspection Duplicates - final byte[] tmpArray = getByteArray(estimateMaxBytes); - int written = StringEncodingUtils.convertUTF16ToUTF8(chars, tmpArray, 0); - long header = ((long) written << 2) | UTF8; - writerIndex += buffer._unsafePutVarUint36Small(writerIndex, header); - buffer.put(writerIndex, tmpArray, 0, written); - buffer._unsafeWriterIndex(writerIndex + written); - } - } - - public void writeCharsUTF8PerfOptimized(MemoryBuffer buffer, char[] chars) { - int estimateMaxBytes = chars.length * 3; - int numBytes = MathUtils.doubleExact(chars.length); - // noinspection Duplicates - int writerIndex = buffer.writerIndex(); - long header = ((long) numBytes << 2) | UTF8; - buffer.ensure(writerIndex + 9 + estimateMaxBytes); - byte[] targetArray = buffer.getHeapMemory(); - if (targetArray != null) { - int targetIndex = buffer._unsafeHeapWriterIndex(); - int arrIndex = targetIndex; - arrIndex += LittleEndian.putVarUint36Small(targetArray, arrIndex, header); - writerIndex += arrIndex - targetIndex; - targetIndex = StringEncodingUtils.convertUTF16ToUTF8(chars, targetArray, arrIndex + 4); - int written = targetIndex - arrIndex - 4; - buffer._unsafePutInt32(writerIndex, written); - buffer._unsafeWriterIndex(writerIndex + 4 + written); - } else { - final byte[] tmpArray = getByteArray(estimateMaxBytes); - int written = StringEncodingUtils.convertUTF16ToUTF8(chars, tmpArray, 0); - writerIndex += buffer._unsafePutVarUint36Small(writerIndex, header); - buffer._unsafePutInt32(writerIndex, written); - writerIndex += 4; - buffer.put(writerIndex, tmpArray, 0, written); - buffer._unsafeWriterIndex(writerIndex + written); - } - } - - private void handleWriteCharsUTF8UnalignedHeaderBytes( - byte[] targetArray, int arrIndex, int diff, int written, byte stashed) { - if (diff == 1) { - System.arraycopy(targetArray, arrIndex + 1, targetArray, arrIndex + 2, written - 1); - targetArray[arrIndex + 1] = stashed; - } else { - System.arraycopy(targetArray, arrIndex, targetArray, arrIndex - 1, written); - } - } - - private void writeBytesUTF8(MemoryBuffer buffer, byte[] bytes) { - int numBytes = bytes.length; - int estimateMaxBytes = bytes.length / 2 * 3; - int writerIndex = buffer.writerIndex(); - buffer.ensure(writerIndex + 9 + estimateMaxBytes); - byte[] targetArray = buffer.getHeapMemory(); - if (targetArray != null) { - // noinspection Duplicates - int targetIndex = buffer._unsafeHeapWriterIndex(); - // keep this index in case actual num utf8 bytes need different bytes for header - int headerPos = targetIndex; - int arrIndex = targetIndex; - long header = ((long) numBytes << 2) | UTF8; - int headerBytesWritten = LittleEndian.putVarUint36Small(targetArray, arrIndex, header); - arrIndex += headerBytesWritten; - writerIndex += arrIndex - targetIndex; - // noinspection Duplicates - targetIndex = StringEncodingUtils.convertUTF16ToUTF8(bytes, targetArray, arrIndex); - byte stashedByte = targetArray[arrIndex]; - int written = targetIndex - arrIndex; - header = ((long) written << 2) | UTF8; - int diff = - LittleEndian.putVarUint36Small(targetArray, headerPos, header) - headerBytesWritten; - if (diff != 0) { - handleWriteCharsUTF8UnalignedHeaderBytes(targetArray, arrIndex, diff, written, stashedByte); - } - buffer._unsafeWriterIndex(writerIndex + written + diff); - } else { - // noinspection Duplicates - final byte[] tmpArray = getByteArray(estimateMaxBytes); - int written = StringEncodingUtils.convertUTF16ToUTF8(bytes, tmpArray, 0); - long header = ((long) written << 2) | UTF8; - writerIndex += buffer._unsafePutVarUint36Small(writerIndex, header); - buffer.put(writerIndex, tmpArray, 0, written); - buffer._unsafeWriterIndex(writerIndex + written); - } - } - - private void writeBytesUTF8PerfOptimized(MemoryBuffer buffer, byte[] bytes) { - int numBytes = bytes.length; - int estimateMaxBytes = bytes.length / 2 * 3; - int writerIndex = buffer.writerIndex(); - long header = ((long) numBytes << 2) | UTF8; - buffer.ensure(writerIndex + 9 + estimateMaxBytes); - byte[] targetArray = buffer.getHeapMemory(); - if (targetArray != null) { - int targetIndex = buffer._unsafeHeapWriterIndex(); - int arrIndex = targetIndex; - arrIndex += LittleEndian.putVarUint36Small(targetArray, arrIndex, header); - writerIndex += arrIndex - targetIndex; - targetIndex = StringEncodingUtils.convertUTF16ToUTF8(bytes, targetArray, arrIndex + 4); - int written = targetIndex - arrIndex - 4; - buffer._unsafePutInt32(writerIndex, written); - buffer._unsafeWriterIndex(writerIndex + 4 + written); - } else { - final byte[] tmpArray = getByteArray(estimateMaxBytes); - int written = StringEncodingUtils.convertUTF16ToUTF8(bytes, tmpArray, 0); - writerIndex += buffer._unsafePutVarUint36Small(writerIndex, header); - buffer._unsafePutInt32(writerIndex, written); - writerIndex += 4; - buffer.put(writerIndex, tmpArray, 0, written); - buffer._unsafeWriterIndex(writerIndex + written); - } - } - - public static String newCharsStringZeroCopy(char[] data) { - if (!jdkInternalFieldAccess()) { - return newCharsStringSlow(data); - } - return _JDKAccess.newCharsStringZeroCopy(data); - } - - private static String newCharsStringSlow(char[] data) { - return new String(data); - } - - // coder param first to make inline call args - // `(buffer.readByte(), buffer.readBytesWithSizeEmbedded())` work. - public static String newBytesStringZeroCopy(byte coder, byte[] data) { - if (!jdkInternalFieldAccess()) { - return newBytesStringSlow(coder, data); - } - return _JDKAccess.newBytesStringZeroCopy(coder, data); - } - - private static String newBytesStringSlow(byte coder, byte[] data) { - if (coder == LATIN1) { - return new String(data, StandardCharsets.ISO_8859_1); - } else if (coder == UTF16) { - char[] chars = new char[data.length >> 1]; - for (int i = 0, j = 0; i < data.length; i += 2) { - chars[j++] = (char) ((data[i] & 0xff) | ((data[i + 1] & 0xff) << 8)); - } - return new String(chars); - } else { - return new String(data, StandardCharsets.UTF_8); - } - } - - private static void writeCharsUTF16BEToHeap( - char[] chars, int arrIndex, int numBytes, byte[] targetArray) { - // Write to heap memory then copy is 250% faster than unsafe write to direct memory. - int charIndex = 0; - for (int i = arrIndex, end = i + numBytes; i < end; i += 2) { - char c = chars[charIndex++]; - targetArray[i] = (byte) c; - targetArray[i + 1] = (byte) (c >>> 8); - } - } - - private static void writeCharsUTF16ToHeapSlow( - char[] chars, int arrIndex, int numBytes, byte[] targetArray) { - writeCharsUTF16BEToHeap(chars, arrIndex, numBytes, targetArray); - } - - private int offHeapWriteCharsUTF16( - MemoryBuffer buffer, char[] chars, int writerIndex, int numBytes) { - byte[] tmpArray = getByteArray(numBytes); - int charIndex = 0; - for (int i = 0; i < numBytes; i += 2) { - char c = chars[charIndex++]; - tmpArray[i] = (byte) (c >> StringUTF16.HI_BYTE_SHIFT); - tmpArray[i + 1] = (byte) (c >> StringUTF16.LO_BYTE_SHIFT); - } - buffer.put(writerIndex, tmpArray, 0, numBytes); - writerIndex += numBytes; - return writerIndex; - } - - private int offHeapWriteCharsUTF16BE( - MemoryBuffer buffer, char[] chars, int writerIndex, int numBytes) { - byte[] tmpArray = getByteArray(numBytes); - int charIndex = 0; - for (int i = 0; i < numBytes; i += 2) { - char c = chars[charIndex++]; - tmpArray[i] = (byte) c; - tmpArray[i + 1] = (byte) (c >>> 8); - } - buffer.put(writerIndex, tmpArray, 0, numBytes); - writerIndex += numBytes; - return writerIndex; - } - - private char[] readCharsUTF16BE(MemoryBuffer buffer, int numBytes) { - buffer.checkReadableBytes(numBytes); - final byte[] targetArray = buffer.getHeapMemory(); - char[] chars = new char[numBytes >> 1]; - if (targetArray != null) { - int charIndex = 0; - for (int i = buffer._unsafeHeapReaderIndex(), end = i + numBytes; i < end; i += 2) { - int lo = targetArray[i] & 0xff; - int hi = targetArray[i + 1] & 0xff; - chars[charIndex++] = (char) (lo | (hi << 8)); - } - buffer._increaseReaderIndexUnsafe(numBytes); - } else { - final byte[] tmpArray = getByteArray(numBytes); - buffer.readBytes(tmpArray, 0, numBytes); - int charIndex = 0; - for (int i = 0; i < numBytes; i += 2) { - int lo = tmpArray[i] & 0xff; - int hi = tmpArray[i + 1] & 0xff; - chars[charIndex++] = (char) (lo | (hi << 8)); - } - } - return chars; - } - - private byte[] readBytesUTF16BE(MemoryBuffer buffer, int numBytes) { - byte[] bytes = readBytesUnCompressedUTF16(buffer, numBytes); - swapUTF16BytesInPlace(bytes); - return bytes; - } - - private static void swapUTF16BytesInPlace(byte[] bytes) { - for (int i = 0; i < bytes.length; i += 2) { - byte tmp = bytes[i]; - bytes[i] = bytes[i + 1]; - bytes[i + 1] = tmp; - } - } - - private static void writeBytesStringUTF16BE(MemoryBuffer buffer, byte[] bytes) { - int bytesLen = bytes.length; - long header = ((long) bytesLen << 2) | UTF16; - int writerIndex = buffer.writerIndex(); - buffer.ensure(writerIndex + 9 + bytesLen); - final byte[] targetArray = buffer.getHeapMemory(); - if (targetArray != null) { - final int targetIndex = buffer._unsafeHeapWriterIndex(); - int arrIndex = targetIndex; - arrIndex += LittleEndian.putVarUint36Small(targetArray, arrIndex, header); - writerIndex += arrIndex - targetIndex; - for (int i = 0; i < bytesLen; i += 2) { - targetArray[arrIndex + i] = bytes[i + 1]; - targetArray[arrIndex + i + 1] = bytes[i]; - } - } else { - writerIndex += buffer._unsafePutVarUint36Small(writerIndex, header); - byte[] tmpArray = new byte[bytesLen]; - for (int i = 0; i < bytesLen; i += 2) { - tmpArray[i] = bytes[i + 1]; - tmpArray[i + 1] = bytes[i]; - } - buffer.put(writerIndex, tmpArray, 0, bytesLen); - } - buffer._unsafeWriterIndex(writerIndex + bytesLen); - } - - private static byte bestCoder(char[] chars) { - int numChars = chars.length; - // sample 64 chars - int sampleNum = Math.min(64, numChars); - int vectorizedLen = sampleNum >> 2; - int vectorizedChars = vectorizedLen << 2; - int endOffset = UnsafeOps.CHAR_ARRAY_OFFSET + (vectorizedChars << 1); - int asciiCount = 0; - int latin1Count = 0; - for (int offset = UnsafeOps.CHAR_ARRAY_OFFSET, charOffset = 0; - offset < endOffset; - offset += 8, charOffset += 4) { - long multiChars = UnsafeOps.getLong(chars, offset); - if ((multiChars & MULTI_CHARS_NON_ASCII_MASK) == 0) { - latin1Count += 4; - asciiCount += 4; - } else if ((multiChars & MULTI_CHARS_NON_LATIN_MASK) == 0) { - latin1Count += 4; - for (int i = 0; i < 4; ++i) { - if (chars[charOffset + i] < 0x80) { - asciiCount++; - } - } - } else { - for (int i = 0; i < 4; ++i) { - if (chars[charOffset + i] < 0x80) { - latin1Count++; - asciiCount++; - } else if (chars[charOffset + i] <= 0xFF) { - latin1Count++; - } - } - } - } - - for (int i = vectorizedChars; i < sampleNum; i++) { - if (chars[i] < 0x80) { - latin1Count++; - asciiCount++; - } else if (chars[i] <= 0xFF) { - latin1Count++; - } - } - - if (latin1Count == numChars - || (latin1Count == sampleNum && StringUtils.isLatin(chars, sampleNum))) { - return LATIN1; - } else if (asciiCount >= sampleNum * 0.5) { - // ascii number > 50%, choose UTF-8 - return UTF8; - } else { - return UTF16; - } - } - - private static byte bestCoder(byte[] bytes) { - int numBytes = bytes.length; - // sample 64 chars - int sampleNum = Math.min(64 << 1, numBytes); - int vectorizedLen = sampleNum >> 3; - int vectorizedBytes = vectorizedLen << 3; - int endOffset = UnsafeOps.BYTE_ARRAY_OFFSET + vectorizedBytes; - int asciiCount = 0; - for (int offset = UnsafeOps.BYTE_ARRAY_OFFSET, bytesOffset = 0; - offset < endOffset; - offset += 8, bytesOffset += 8) { - long multiChars = UnsafeOps.getLong(bytes, offset); - if ((multiChars & MULTI_CHARS_NON_ASCII_MASK) == 0) { - asciiCount += 4; - } else { - for (int i = 0; i < 8; i += 2) { - if (UnsafeOps.getChar(bytes, offset + i) < 0x80) { - asciiCount++; - } - } - } - } - for (int i = vectorizedBytes; vectorizedBytes < sampleNum; vectorizedBytes += 2) { - if (UnsafeOps.getChar(bytes, UnsafeOps.BYTE_ARRAY_OFFSET + i) < 0x80) { - asciiCount++; - } - } - // ascii number > 50%, choose UTF-8 - if (asciiCount >= sampleNum * 0.5) { - return UTF8; - } else { - return UTF16; - } - } - - private char[] getCharArray(int numElements) { - char[] charArray = this.charArray; - if (charArray.length < numElements) { - charArray = new char[numElements]; - this.charArray = charArray; - } - if (charArray.length > DEFAULT_BUFFER_SIZE) { - smoothCharArrayLength = - Math.max(((int) (smoothCharArrayLength * 0.9 + numElements * 0.1)), DEFAULT_BUFFER_SIZE); - if (smoothByteArrayLength <= DEFAULT_BUFFER_SIZE) { - this.charArray = new char[DEFAULT_BUFFER_SIZE]; - } - } - return charArray; - } - - byte[] getByteArray(int numElements) { - byte[] byteArray = this.byteArray; - if (byteArray.length < numElements) { - byteArray = new byte[numElements]; - this.byteArray = byteArray; - } - if (byteArray.length > DEFAULT_BUFFER_SIZE) { - smoothByteArrayLength = - Math.max(((int) (smoothByteArrayLength * 0.9 + numElements * 0.1)), DEFAULT_BUFFER_SIZE); - if (smoothByteArrayLength <= DEFAULT_BUFFER_SIZE) { - this.byteArray = new byte[DEFAULT_BUFFER_SIZE]; - } - } - return byteArray; - } - - private byte[] getByteArray2(int numElements) { - byte[] byteArray2 = this.byteArray2; - if (byteArray2.length < numElements) { - byteArray2 = new byte[numElements]; - this.byteArray = byteArray2; - } - if (byteArray2.length > DEFAULT_BUFFER_SIZE) { - smoothByteArrayLength = - Math.max(((int) (smoothByteArrayLength * 0.9 + numElements * 0.1)), DEFAULT_BUFFER_SIZE); - if (smoothByteArrayLength <= DEFAULT_BUFFER_SIZE) { - this.byteArray2 = new byte[DEFAULT_BUFFER_SIZE]; - } - } - return byteArray2; - } - - public void clearBuffer(int size) { - if (byteArray.length >= size) { - byteArray = EMPTY_BYTES_STUB; - } - if (byteArray2.length >= size) { - byteArray2 = EMPTY_BYTES_STUB; - } - if (charArray.length >= size) { - this.charArray = EMPTY_CHARS_STUB; - } - } -} diff --git a/java/fory-core/src/test/java/org/apache/fory/util/StringEncodingUtilsTest.java b/java/fory-core/src/test/java/org/apache/fory/serializer/StringEncodingUtilsTest.java similarity index 98% rename from java/fory-core/src/test/java/org/apache/fory/util/StringEncodingUtilsTest.java rename to java/fory-core/src/test/java/org/apache/fory/serializer/StringEncodingUtilsTest.java index 769e96c0d8..7df39320e1 100644 --- a/java/fory-core/src/test/java/org/apache/fory/util/StringEncodingUtilsTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/serializer/StringEncodingUtilsTest.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.fory.util; +package org.apache.fory.serializer; import static org.testng.Assert.assertEquals; diff --git a/java/fory-core/src/test/java/org/apache/fory/util/StringUtilsTest.java b/java/fory-core/src/test/java/org/apache/fory/util/StringUtilsTest.java index 7aa1893ee7..4d639a4236 100644 --- a/java/fory-core/src/test/java/org/apache/fory/util/StringUtilsTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/util/StringUtilsTest.java @@ -26,6 +26,7 @@ import org.apache.fory.ForyTestBase; import org.apache.fory.memory.NativeByteOrder; import org.apache.fory.platform.UnsafeOps; +import org.apache.fory.serializer.StringEncodingUtils; import org.testng.annotations.Test; public class StringUtilsTest extends ForyTestBase { @@ -153,23 +154,23 @@ private boolean isLatin(char[] chars, boolean isLittle) { @Test public void testLatinCheck() { - assertTrue(StringUtils.isLatin("Fory".toCharArray())); - assertTrue(StringUtils.isLatin(StringUtils.random(8 * 10).toCharArray())); + assertTrue(StringEncodingUtils.isLatin("Fory".toCharArray())); + assertTrue(StringEncodingUtils.isLatin(StringUtils.random(8 * 10).toCharArray())); // test unaligned - assertTrue(StringUtils.isLatin((StringUtils.random(8 * 10) + "1").toCharArray())); - assertTrue(StringUtils.isLatin((StringUtils.random(8 * 10) + "12").toCharArray())); - assertTrue(StringUtils.isLatin((StringUtils.random(8 * 10) + "123").toCharArray())); - assertFalse(StringUtils.isLatin("你好, Fory".toCharArray())); - assertFalse(StringUtils.isLatin((StringUtils.random(8 * 10) + "你好").toCharArray())); - assertFalse(StringUtils.isLatin((StringUtils.random(8 * 10) + "1你好").toCharArray())); - assertFalse(StringUtils.isLatin((StringUtils.random(11) + "你").toCharArray())); - assertFalse(StringUtils.isLatin((StringUtils.random(10) + "你好").toCharArray())); - assertFalse(StringUtils.isLatin((StringUtils.random(9) + "性能好").toCharArray())); - assertFalse(StringUtils.isLatin("\u1234".toCharArray())); - assertFalse(StringUtils.isLatin("a\u1234".toCharArray())); - assertFalse(StringUtils.isLatin("ab\u1234".toCharArray())); - assertFalse(StringUtils.isLatin("abc\u1234".toCharArray())); - assertFalse(StringUtils.isLatin("abcd\u1234".toCharArray())); - assertFalse(StringUtils.isLatin("Javaone Keynote\u1234".toCharArray())); + assertTrue(StringEncodingUtils.isLatin((StringUtils.random(8 * 10) + "1").toCharArray())); + assertTrue(StringEncodingUtils.isLatin((StringUtils.random(8 * 10) + "12").toCharArray())); + assertTrue(StringEncodingUtils.isLatin((StringUtils.random(8 * 10) + "123").toCharArray())); + assertFalse(StringEncodingUtils.isLatin("你好, Fory".toCharArray())); + assertFalse(StringEncodingUtils.isLatin((StringUtils.random(8 * 10) + "你好").toCharArray())); + assertFalse(StringEncodingUtils.isLatin((StringUtils.random(8 * 10) + "1你好").toCharArray())); + assertFalse(StringEncodingUtils.isLatin((StringUtils.random(11) + "你").toCharArray())); + assertFalse(StringEncodingUtils.isLatin((StringUtils.random(10) + "你好").toCharArray())); + assertFalse(StringEncodingUtils.isLatin((StringUtils.random(9) + "性能好").toCharArray())); + assertFalse(StringEncodingUtils.isLatin("\u1234".toCharArray())); + assertFalse(StringEncodingUtils.isLatin("a\u1234".toCharArray())); + assertFalse(StringEncodingUtils.isLatin("ab\u1234".toCharArray())); + assertFalse(StringEncodingUtils.isLatin("abc\u1234".toCharArray())); + assertFalse(StringEncodingUtils.isLatin("abcd\u1234".toCharArray())); + assertFalse(StringEncodingUtils.isLatin("Javaone Keynote\u1234".toCharArray())); } } From d602cbfc02b51f7ad2ce186fb74e50c5f6c7743e Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sun, 24 May 2026 17:46:30 +0800 Subject: [PATCH 26/69] refactor(java): remove versioned lambda serializer --- java/fory-core/pom.xml | 7 - .../fory/platform/internal/_JDKAccess.java | 5 + .../SerializedLambdaSerializer.java | 3 +- .../fory/platform/internal/_JDKAccess.java | 24 ++ .../SerializedLambdaSerializer.java | 216 ------------------ 5 files changed, 30 insertions(+), 225 deletions(-) delete mode 100644 java/fory-core/src/main/java25/org/apache/fory/serializer/SerializedLambdaSerializer.java diff --git a/java/fory-core/pom.xml b/java/fory-core/pom.xml index 99ccc7a058..4fbcb35b27 100644 --- a/java/fory-core/pom.xml +++ b/java/fory-core/pom.xml @@ -250,7 +250,6 @@ - @@ -282,9 +281,6 @@ - @@ -318,9 +314,6 @@ - diff --git a/java/fory-core/src/main/java/org/apache/fory/platform/internal/_JDKAccess.java b/java/fory-core/src/main/java/org/apache/fory/platform/internal/_JDKAccess.java index 7019213ed0..8584f722f4 100644 --- a/java/fory-core/src/main/java/org/apache/fory/platform/internal/_JDKAccess.java +++ b/java/fory-core/src/main/java/org/apache/fory/platform/internal/_JDKAccess.java @@ -220,6 +220,11 @@ public static Lookup _trustedLookup(Class objectClass) { return lookupCache.get(objectClass, () -> _Lookup._trustedLookup(objectClass)); } + public static MethodHandle readResolveHandle(Class cls, Method method) + throws IllegalAccessException { + return _trustedLookup(cls).unreflect(method); + } + private static final byte LATIN1 = 0; private static final Byte LATIN1_BOXED = LATIN1; private static final byte UTF16 = 1; diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/SerializedLambdaSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/SerializedLambdaSerializer.java index ec8c3a604e..c9897b8442 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/SerializedLambdaSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/SerializedLambdaSerializer.java @@ -51,8 +51,7 @@ public class SerializedLambdaSerializer extends Serializer { Method readResolveMethod = JavaSerializer.getReadResolveMethod(SERIALIZED_LAMBDA); Preconditions.checkNotNull( readResolveMethod, "Missing readResolve for " + SERIALIZED_LAMBDA); - READ_RESOLVE_HANDLE = - _JDKAccess._trustedLookup(SERIALIZED_LAMBDA).unreflect(readResolveMethod); + READ_RESOLVE_HANDLE = _JDKAccess.readResolveHandle(SERIALIZED_LAMBDA, readResolveMethod); } catch (IllegalAccessException e) { throw new ForyException(e); } diff --git a/java/fory-core/src/main/java25/org/apache/fory/platform/internal/_JDKAccess.java b/java/fory-core/src/main/java25/org/apache/fory/platform/internal/_JDKAccess.java index 306b782642..0c057eb336 100644 --- a/java/fory-core/src/main/java25/org/apache/fory/platform/internal/_JDKAccess.java +++ b/java/fory-core/src/main/java25/org/apache/fory/platform/internal/_JDKAccess.java @@ -29,6 +29,7 @@ import java.lang.invoke.MethodHandles; import java.lang.invoke.MethodHandles.Lookup; import java.lang.invoke.MethodType; +import java.lang.invoke.SerializedLambda; import java.lang.invoke.VarHandle; import java.lang.reflect.Field; import java.lang.reflect.Method; @@ -341,6 +342,29 @@ public static Lookup _trustedLookup(Class objectClass) { return lookupCache.get(objectClass, () -> _Lookup._trustedLookup(objectClass)); } + public static MethodHandle readResolveHandle(Class cls, Method method) + throws IllegalAccessException { + try { + return _trustedLookup(cls).unreflect(method); + } catch (IllegalArgumentException e) { + if (cls != SerializedLambda.class) { + throw e; + } + // JDK25 rejects SerializedLambda itself as a privateLookupIn target. Reflective access still + // honors the same java.base/java.lang.invoke open requirement and avoids serializer-level + // versioning. + try { + method.setAccessible(true); + } catch (RuntimeException inaccessible) { + throw new IllegalStateException( + "SerializedLambda readResolve requires java.base/java.lang.invoke to be open to " + + "org.apache.fory.core,org.apache.fory.format", + inaccessible); + } + return MethodHandles.lookup().unreflect(method); + } + } + private static final byte LATIN1 = 0; private static final Byte LATIN1_BOXED = LATIN1; private static final byte UTF16 = 1; diff --git a/java/fory-core/src/main/java25/org/apache/fory/serializer/SerializedLambdaSerializer.java b/java/fory-core/src/main/java25/org/apache/fory/serializer/SerializedLambdaSerializer.java deleted file mode 100644 index a744b99fff..0000000000 --- a/java/fory-core/src/main/java25/org/apache/fory/serializer/SerializedLambdaSerializer.java +++ /dev/null @@ -1,216 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.fory.serializer; - -import java.lang.invoke.MethodHandle; -import java.lang.invoke.SerializedLambda; -import java.lang.reflect.Method; -import org.apache.fory.context.CopyContext; -import org.apache.fory.context.ReadContext; -import org.apache.fory.context.WriteContext; -import org.apache.fory.exception.ForyException; -import org.apache.fory.memory.MemoryBuffer; -import org.apache.fory.platform.AndroidSupport; -import org.apache.fory.platform.internal._JDKAccess; -import org.apache.fory.resolver.TypeResolver; -import org.apache.fory.util.Preconditions; - -/** - * Serializer for {@link SerializedLambda}. It writes the JDK lambda payload through the public - * getter API, applies {@code readResolve} on read, and preserves unresolved {@code - * SerializedLambda} form on direct copy. - */ -@SuppressWarnings({"rawtypes", "unchecked"}) -public class SerializedLambdaSerializer extends Serializer { - static final Class SERIALIZED_LAMBDA = SerializedLambda.class; - private static final MethodHandle READ_RESOLVE_HANDLE; - private final TypeResolver typeResolver; - - static { - MethodHandle readResolveHandle = null; - if (AndroidSupport.IS_ANDROID) { - // Lambda serialization is unsupported on Android. - } else { - try { - Method readResolveMethod = JavaSerializer.getReadResolveMethod(SERIALIZED_LAMBDA); - Preconditions.checkNotNull( - readResolveMethod, "Missing readResolve for " + SERIALIZED_LAMBDA); - try { - readResolveHandle = - _JDKAccess._trustedLookup(SERIALIZED_LAMBDA).unreflect(readResolveMethod); - } catch (RuntimeException e) { - // JDK25 rejects privateLookupIn for java.lang.invoke.SerializedLambda itself. With - // java.lang.invoke opened, reflective access still exposes the JDK readResolve method - // without using Unsafe. - readResolveMethod.setAccessible(true); - readResolveHandle = java.lang.invoke.MethodHandles.lookup().unreflect(readResolveMethod); - } - } catch (IllegalAccessException | RuntimeException e) { - // Keep serializer registration available; readResolve reports the missing open if used. - } - } - READ_RESOLVE_HANDLE = readResolveHandle; - } - - public SerializedLambdaSerializer(TypeResolver typeResolver, Class cls) { - super(typeResolver.getConfig(), cls); - this.typeResolver = typeResolver; - Preconditions.checkArgument(cls == SERIALIZED_LAMBDA); - } - - @Override - public void write(WriteContext writeContext, Object value) { - throwIfAndroid(); - MemoryBuffer buffer = writeContext.getBuffer(); - SerializedLambda serializedLambda = (SerializedLambda) value; - writeContext.writeStringRef(serializedLambda.getCapturingClass()); - writeContext.writeStringRef(serializedLambda.getFunctionalInterfaceClass()); - writeContext.writeStringRef(serializedLambda.getFunctionalInterfaceMethodName()); - writeContext.writeStringRef(serializedLambda.getFunctionalInterfaceMethodSignature()); - writeContext.writeStringRef(serializedLambda.getImplClass()); - writeContext.writeStringRef(serializedLambda.getImplMethodName()); - writeContext.writeStringRef(serializedLambda.getImplMethodSignature()); - buffer.writeVarInt32(serializedLambda.getImplMethodKind()); - writeContext.writeStringRef(serializedLambda.getInstantiatedMethodType()); - int capturedArgCount = serializedLambda.getCapturedArgCount(); - buffer.writeVarUInt32Small7(capturedArgCount); - for (int i = 0; i < capturedArgCount; i++) { - writeContext.writeRef(serializedLambda.getCapturedArg(i)); - } - } - - @Override - public Object copy(CopyContext copyContext, Object value) { - throwIfAndroid(); - SerializedLambda serializedLambda = (SerializedLambda) value; - int capturedArgCount = serializedLambda.getCapturedArgCount(); - Object[] capturedArgs = new Object[capturedArgCount]; - for (int i = 0; i < capturedArgCount; i++) { - capturedArgs[i] = copyContext.copyObject(serializedLambda.getCapturedArg(i)); - } - return newSerializedLambda( - serializedLambda.getCapturingClass(), - serializedLambda.getFunctionalInterfaceClass(), - serializedLambda.getFunctionalInterfaceMethodName(), - serializedLambda.getFunctionalInterfaceMethodSignature(), - serializedLambda.getImplMethodKind(), - serializedLambda.getImplClass(), - serializedLambda.getImplMethodName(), - serializedLambda.getImplMethodSignature(), - serializedLambda.getInstantiatedMethodType(), - capturedArgs); - } - - @Override - public Object read(ReadContext readContext) { - throwIfAndroid(); - return readResolve(readUnresolved(readContext)); - } - - Object readUnresolved(ReadContext readContext) { - throwIfAndroid(); - MemoryBuffer buffer = readContext.getBuffer(); - String capturingClass = readContext.readStringRef(); - String functionalInterfaceClass = readContext.readStringRef(); - String functionalInterfaceMethodName = readContext.readStringRef(); - String functionalInterfaceMethodSignature = readContext.readStringRef(); - String implClass = readContext.readStringRef(); - String implMethodName = readContext.readStringRef(); - String implMethodSignature = readContext.readStringRef(); - int implMethodKind = buffer.readVarInt32(); - String instantiatedMethodType = readContext.readStringRef(); - int capturedArgCount = buffer.readVarUInt32Small7(); - Object[] capturedArgs = new Object[capturedArgCount]; - for (int i = 0; i < capturedArgCount; i++) { - capturedArgs[i] = readContext.readRef(); - } - return newSerializedLambda( - capturingClass, - functionalInterfaceClass, - functionalInterfaceMethodName, - functionalInterfaceMethodSignature, - implMethodKind, - implClass, - implMethodName, - implMethodSignature, - instantiatedMethodType, - capturedArgs); - } - - static Object readResolve(Object replacement) { - throwIfAndroid(); - if (READ_RESOLVE_HANDLE == null) { - throw new ForyException( - "SerializedLambda.readResolve is inaccessible. On JDK25+, deserialize lambdas only " - + "with --add-opens=java.base/java.lang.invoke=" - + "org.apache.fory.core,org.apache.fory.format."); - } - try { - return READ_RESOLVE_HANDLE.invoke(replacement); - } catch (Throwable e) { - throw new RuntimeException("Can't deserialize lambda", e); - } - } - - private static void throwIfAndroid() { - if (AndroidSupport.IS_ANDROID) { - throw new UnsupportedOperationException( - "Lambda serialization is unsupported on Android; serialize explicit data objects instead."); - } - } - - private SerializedLambda newSerializedLambda( - String capturingClass, - String functionalInterfaceClass, - String functionalInterfaceMethodName, - String functionalInterfaceMethodSignature, - int implMethodKind, - String implClass, - String implMethodName, - String implMethodSignature, - String instantiatedMethodType, - Object[] capturedArgs) { - return new SerializedLambda( - loadCapturingClass(capturingClass), - functionalInterfaceClass, - functionalInterfaceMethodName, - functionalInterfaceMethodSignature, - implMethodKind, - implClass, - implMethodName, - implMethodSignature, - instantiatedMethodType, - capturedArgs); - } - - private Class loadCapturingClass(String className) { - String binaryClassName = className.replace('/', '.'); - try { - return Class.forName(binaryClassName, false, typeResolver.getClassLoader()); - } catch (ClassNotFoundException e) { - try { - return Class.forName( - binaryClassName, false, Thread.currentThread().getContextClassLoader()); - } catch (ClassNotFoundException ex) { - throw new RuntimeException("Can't load capturing class " + binaryClassName, ex); - } - } - } -} From a17ab3fd824ae15d06efc8c9fdf84150a558f630 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sun, 24 May 2026 17:55:46 +0800 Subject: [PATCH 27/69] refactor(java): remove versioned serializers registry --- benchmarks/java/pom.xml | 34 +- java/fory-core/pom.xml | 7 - .../apache/fory/serializer/Serializers.java | 864 ------------------ 3 files changed, 5 insertions(+), 900 deletions(-) delete mode 100644 java/fory-core/src/main/java25/org/apache/fory/serializer/Serializers.java diff --git a/benchmarks/java/pom.xml b/benchmarks/java/pom.xml index 07b793301b..2c8e5f33a1 100644 --- a/benchmarks/java/pom.xml +++ b/benchmarks/java/pom.xml @@ -279,13 +279,7 @@ - - - + name="META-INF/versions/25/org/apache/fory/serializer/PlatformStringUtils.class"/> - - - + file="${jdk25.benchmark.check.dir}/META-INF/versions/25/org/apache/fory/serializer/PlatformStringUtils.class" + property="jdk25.benchmark.platformstring.present"/> @@ -343,17 +328,8 @@ unless="jdk25.benchmark.objectcodecbuilder.present" message="JDK25 benchmark jar is missing the versioned ObjectCodecBuilder class."/> - - - + unless="jdk25.benchmark.platformstring.present" + message="JDK25 benchmark jar is missing the versioned PlatformStringUtils class."/> - @@ -281,9 +280,6 @@ - @@ -314,9 +310,6 @@ - diff --git a/java/fory-core/src/main/java25/org/apache/fory/serializer/Serializers.java b/java/fory-core/src/main/java25/org/apache/fory/serializer/Serializers.java deleted file mode 100644 index 627461be29..0000000000 --- a/java/fory-core/src/main/java25/org/apache/fory/serializer/Serializers.java +++ /dev/null @@ -1,864 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.fory.serializer; - -import static org.apache.fory.util.function.Functions.makeGetterFunction; - -import java.lang.invoke.MethodHandle; -import java.lang.invoke.MethodHandles; -import java.lang.invoke.MethodType; -import java.lang.reflect.Constructor; -import java.lang.reflect.Method; -import java.lang.reflect.Modifier; -import java.math.BigDecimal; -import java.math.BigInteger; -import java.net.URI; -import java.nio.charset.Charset; -import java.util.Currency; -import java.util.StringTokenizer; -import java.util.UUID; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.atomic.AtomicReference; -import java.util.function.Function; -import java.util.function.ToIntFunction; -import java.util.regex.Pattern; -import org.apache.fory.Fory; -import org.apache.fory.collection.Cache; -import org.apache.fory.collection.CacheBuilder; -import org.apache.fory.collection.Tuple2; -import org.apache.fory.config.Config; -import org.apache.fory.context.CopyContext; -import org.apache.fory.context.ReadContext; -import org.apache.fory.context.WriteContext; -import org.apache.fory.memory.MemoryBuffer; -import org.apache.fory.memory.MemoryUtils; -import org.apache.fory.meta.TypeDef; -import org.apache.fory.platform.AndroidSupport; -import org.apache.fory.platform.GraalvmSupport; -import org.apache.fory.platform.internal._JDKAccess; -import org.apache.fory.reflect.FieldAccessor; -import org.apache.fory.reflect.ReflectionUtils; -import org.apache.fory.resolver.ClassResolver; -import org.apache.fory.resolver.TypeInfo; -import org.apache.fory.resolver.TypeResolver; -import org.apache.fory.serializer.CodegenSerializer.LazyInitBeanSerializer; -import org.apache.fory.serializer.collection.ChildContainerSerializers; -import org.apache.fory.serializer.collection.CollectionSerializer; -import org.apache.fory.serializer.collection.CollectionSerializers; -import org.apache.fory.serializer.collection.MapSerializer; -import org.apache.fory.serializer.collection.MapSerializers; -import org.apache.fory.serializer.scala.SingletonCollectionSerializer; -import org.apache.fory.serializer.scala.SingletonMapSerializer; -import org.apache.fory.serializer.scala.SingletonObjectSerializer; -import org.apache.fory.util.ExceptionUtils; - -/** Serialization utils and common serializers. */ -@SuppressWarnings({"rawtypes", "unchecked"}) -public class Serializers { - // avoid duplicate reflect inspection and cache for graalvm support too. - private static final Cache> CTR_MAP; - - static { - if (GraalvmSupport.isGraalBuildTime()) { - CTR_MAP = CacheBuilder.newBuilder().concurrencyLevel(32).build(); - } else { - CTR_MAP = CacheBuilder.newBuilder().weakKeys().softValues().build(); - } - } - - private static final MethodType SIG1 = - MethodType.methodType(void.class, TypeResolver.class, Class.class); - private static final MethodType SIG2 = MethodType.methodType(void.class, TypeResolver.class); - private static final MethodType SIG3 = - MethodType.methodType(void.class, Config.class, Class.class); - private static final MethodType SIG4 = MethodType.methodType(void.class, Config.class); - private static final MethodType SIG5 = MethodType.methodType(void.class, Class.class); - private static final MethodType SIG6 = MethodType.methodType(void.class); - - /** - * Serializer subclass must have a constructor which take parameters of type {@link TypeResolver} - * and {@link Class}, or {@link TypeResolver}, or {@link Config} and {@link Class}, or {@link - * Config}, or {@link Class}, or no-arg constructor. - */ - public static Serializer newSerializer( - Fory fory, Class type, Class serializerClass) { - return newSerializer(fory.getTypeResolver(), type, serializerClass); - } - - /** - * Serializer subclass must have a constructor which take parameters of type {@link TypeResolver} - * and {@link Class}, or {@link TypeResolver}, or {@link Config} and {@link Class}, or {@link - * Config}, or {@link Class}, or no-arg constructor. - */ - public static Serializer newSerializer( - TypeResolver typeResolver, Class type, Class serializerClass) { - TypeInfo typeInfo = typeResolver.getTypeInfo(type, false); - Serializer serializer = typeInfo == null ? null : typeInfo.getSerializer(); - try { - return buildSerializer(typeResolver, type, serializerClass); - } catch (Throwable t) { - // Some serializer may set itself in constructor as serializer, but the - // constructor failed later. For example, some final type field doesn't - // support serialization. - typeResolver.resetSerializer(type, serializer); - if (t instanceof java.lang.reflect.InvocationTargetException && t.getCause() != null) { - ExceptionUtils.throwException(t.getCause()); - } - ExceptionUtils.throwException(t); - } - throw new IllegalStateException("unreachable"); - } - - private static Serializer buildSerializer( - TypeResolver typeResolver, Class type, Class serializerClass) { - try { - Config config = typeResolver.getConfig(); - Serializer serializer = - buildBuiltinSerializer(typeResolver, config, type, serializerClass); - if (serializer != null) { - return serializer; - } - Tuple2 ctrInfo = CTR_MAP.getIfPresent(serializerClass); - if (ctrInfo != null) { - MethodType sig = ctrInfo.f0; - MethodHandle handle = ctrInfo.f1; - if (sig.equals(SIG1)) { - return (Serializer) handle.invoke(typeResolver, type); - } else if (sig.equals(SIG2)) { - return (Serializer) handle.invoke(typeResolver); - } else if (sig.equals(SIG3)) { - return (Serializer) handle.invoke(config, type); - } else if (sig.equals(SIG4)) { - return (Serializer) handle.invoke(config); - } else if (sig.equals(SIG5)) { - return (Serializer) handle.invoke(type); - } else { - return (Serializer) handle.invoke(); - } - } - return createSerializer(typeResolver, type, serializerClass); - } catch (Throwable t) { - ExceptionUtils.throwException(t); - throw new IllegalStateException("unreachable"); - } - } - - private static Serializer buildBuiltinSerializer( - TypeResolver typeResolver, - Config config, - Class type, - Class serializerClass) { - if (serializerClass == ObjectSerializer.class) { - return new ObjectSerializer(typeResolver, type); - } - if (serializerClass == ArraySerializers.ObjectArraySerializer.class) { - return (Serializer) new ArraySerializers.ObjectArraySerializer(typeResolver, type); - } - if (serializerClass == ObjectStreamSerializer.class) { - return new ObjectStreamSerializer(typeResolver, type); - } - if (serializerClass == ExceptionSerializers.ExceptionSerializer.class) { - return new ExceptionSerializers.ExceptionSerializer(typeResolver, type); - } - if (serializerClass == ExceptionSerializers.StackTraceElementSerializer.class) { - return (Serializer) new ExceptionSerializers.StackTraceElementSerializer(config); - } - if (serializerClass == CompatibleSerializer.class) { - TypeDef typeDef = typeResolver.getTypeDef(type, true); - return new CompatibleSerializer(typeResolver, type, typeDef); - } - if (serializerClass == EnumSerializer.class) { - return (Serializer) new EnumSerializer(config, type); - } - if (serializerClass == LambdaSerializer.class) { - return new LambdaSerializer(typeResolver, type); - } - if (serializerClass == JdkProxySerializer.class) { - return new JdkProxySerializer(typeResolver, type); - } - if (serializerClass == ReplaceResolveSerializer.class) { - return new ReplaceResolveSerializer(typeResolver, type); - } - if (serializerClass == ExternalizableSerializer.class) { - return new ExternalizableSerializer(typeResolver, type); - } - if (serializerClass == LazyInitBeanSerializer.class) { - return new LazyInitBeanSerializer(typeResolver, type); - } - if (serializerClass == TimeSerializers.CalendarSerializer.class) { - return (Serializer) new TimeSerializers.CalendarSerializer(config, type); - } - if (serializerClass == TimeSerializers.ZoneIdSerializer.class) { - return (Serializer) new TimeSerializers.ZoneIdSerializer(config, type); - } - if (serializerClass == TimeSerializers.TimeZoneSerializer.class) { - return (Serializer) new TimeSerializers.TimeZoneSerializer(config, type); - } - if (serializerClass == BufferSerializers.ByteBufferSerializer.class) { - return (Serializer) new BufferSerializers.ByteBufferSerializer(typeResolver, type); - } - if (serializerClass == CharsetSerializer.class) { - return new CharsetSerializer(config, type); - } - if (serializerClass == CollectionSerializers.EnumSetSerializer.class) { - return (Serializer) new CollectionSerializers.EnumSetSerializer(typeResolver, type); - } - if (serializerClass == CollectionSerializer.class) { - return new CollectionSerializer(typeResolver, type); - } - if (serializerClass == CollectionSerializers.DefaultJavaCollectionSerializer.class) { - return new CollectionSerializers.DefaultJavaCollectionSerializer(typeResolver, type); - } - if (serializerClass == CollectionSerializers.JDKCompatibleCollectionSerializer.class) { - return new CollectionSerializers.JDKCompatibleCollectionSerializer(typeResolver, type); - } - if (serializerClass == MapSerializer.class) { - return new MapSerializer(typeResolver, type); - } - if (serializerClass == MapSerializers.DefaultJavaMapSerializer.class) { - return new MapSerializers.DefaultJavaMapSerializer(typeResolver, type); - } - if (serializerClass == MapSerializers.JDKCompatibleMapSerializer.class) { - return new MapSerializers.JDKCompatibleMapSerializer(typeResolver, type); - } - if (serializerClass == ChildContainerSerializers.ChildCollectionSerializer.class) { - return new ChildContainerSerializers.ChildCollectionSerializer(typeResolver, type); - } - if (serializerClass == ChildContainerSerializers.ChildArrayListSerializer.class) { - return new ChildContainerSerializers.ChildArrayListSerializer(typeResolver, type); - } - if (serializerClass == ChildContainerSerializers.ChildMapSerializer.class) { - return new ChildContainerSerializers.ChildMapSerializer(typeResolver, type); - } - if (serializerClass == ChildContainerSerializers.ChildSortedSetSerializer.class) { - return new ChildContainerSerializers.ChildSortedSetSerializer(typeResolver, type); - } - if (serializerClass == ChildContainerSerializers.ChildPriorityQueueSerializer.class) { - return new ChildContainerSerializers.ChildPriorityQueueSerializer(typeResolver, type); - } - if (serializerClass == ChildContainerSerializers.ChildSortedMapSerializer.class) { - return new ChildContainerSerializers.ChildSortedMapSerializer(typeResolver, type); - } - if (serializerClass == SingletonCollectionSerializer.class) { - return new SingletonCollectionSerializer(typeResolver, type); - } - if (serializerClass == SingletonMapSerializer.class) { - return new SingletonMapSerializer(typeResolver, type); - } - if (serializerClass == SingletonObjectSerializer.class) { - return new SingletonObjectSerializer(typeResolver, type); - } - return null; - } - - private static Serializer createSerializer( - TypeResolver typeResolver, Class type, Class serializerClass) { - if (AndroidSupport.IS_ANDROID) { - return createSerializerReflectively(typeResolver, type, serializerClass); - } - try { - Config config = typeResolver.getConfig(); - try { - MethodHandle ctr = findConstructor(serializerClass, SIG1); - CTR_MAP.put(serializerClass, Tuple2.of(SIG1, ctr)); - return (Serializer) ctr.invoke(typeResolver, type); - } catch (NoSuchMethodException e) { - ExceptionUtils.ignore(e); - } - try { - MethodHandle ctr = findConstructor(serializerClass, SIG2); - CTR_MAP.put(serializerClass, Tuple2.of(SIG2, ctr)); - return (Serializer) ctr.invoke(typeResolver); - } catch (NoSuchMethodException e) { - ExceptionUtils.ignore(e); - } - try { - MethodHandle ctr = findConstructor(serializerClass, SIG3); - CTR_MAP.put(serializerClass, Tuple2.of(SIG3, ctr)); - return (Serializer) ctr.invoke(config, type); - } catch (NoSuchMethodException e) { - ExceptionUtils.ignore(e); - } - try { - MethodHandle ctr = findConstructor(serializerClass, SIG4); - CTR_MAP.put(serializerClass, Tuple2.of(SIG4, ctr)); - return (Serializer) ctr.invoke(config); - } catch (NoSuchMethodException e) { - ExceptionUtils.ignore(e); - } - try { - MethodHandle ctr = findConstructor(serializerClass, SIG5); - CTR_MAP.put(serializerClass, Tuple2.of(SIG5, ctr)); - return (Serializer) ctr.invoke(type); - } catch (NoSuchMethodException e) { - ExceptionUtils.ignore(e); - } - MethodHandle ctr = ReflectionUtils.getCtrHandle(serializerClass); - CTR_MAP.put(serializerClass, Tuple2.of(SIG6, ctr)); - return (Serializer) ctr.invoke(); - } catch (Throwable t) { - ExceptionUtils.throwException(t); - throw new IllegalStateException("unreachable"); - } - } - - private static MethodHandle findConstructor(Class cls, MethodType sig) - throws NoSuchMethodException, IllegalAccessException { - if (Modifier.isPublic(cls.getModifiers())) { - try { - return MethodHandles.publicLookup().findConstructor(cls, sig); - } catch (IllegalAccessException ignored) { - // The class may be public in a non-exported package. Fall back to the private lookup path so - // named-module users can still enable access with opens. - } - } - return _JDKAccess._trustedLookup(cls).findConstructor(cls, sig); - } - - private static Serializer createSerializerReflectively( - TypeResolver typeResolver, Class type, Class serializerClass) { - Config config = typeResolver.getConfig(); - try { - Constructor ctr = - serializerClass.getDeclaredConstructor(TypeResolver.class, Class.class); - ctr.setAccessible(true); - return (Serializer) ctr.newInstance(typeResolver, type); - } catch (NoSuchMethodException e) { - ExceptionUtils.ignore(e); - } catch (Throwable t) { - ExceptionUtils.throwException(t); - throw new IllegalStateException("unreachable"); - } - try { - Constructor ctr = - serializerClass.getDeclaredConstructor(TypeResolver.class); - ctr.setAccessible(true); - return (Serializer) ctr.newInstance(typeResolver); - } catch (NoSuchMethodException e) { - ExceptionUtils.ignore(e); - } catch (Throwable t) { - ExceptionUtils.throwException(t); - throw new IllegalStateException("unreachable"); - } - try { - Constructor ctr = - serializerClass.getDeclaredConstructor(Config.class, Class.class); - ctr.setAccessible(true); - return (Serializer) ctr.newInstance(config, type); - } catch (NoSuchMethodException e) { - ExceptionUtils.ignore(e); - } catch (Throwable t) { - ExceptionUtils.throwException(t); - throw new IllegalStateException("unreachable"); - } - try { - Constructor ctr = serializerClass.getDeclaredConstructor(Config.class); - ctr.setAccessible(true); - return (Serializer) ctr.newInstance(config); - } catch (NoSuchMethodException e) { - ExceptionUtils.ignore(e); - } catch (Throwable t) { - ExceptionUtils.throwException(t); - throw new IllegalStateException("unreachable"); - } - try { - Constructor ctr = serializerClass.getDeclaredConstructor(Class.class); - ctr.setAccessible(true); - return (Serializer) ctr.newInstance(type); - } catch (NoSuchMethodException e) { - ExceptionUtils.ignore(e); - } catch (Throwable t) { - ExceptionUtils.throwException(t); - throw new IllegalStateException("unreachable"); - } - try { - Constructor ctr = serializerClass.getDeclaredConstructor(); - ctr.setAccessible(true); - return (Serializer) ctr.newInstance(); - } catch (NoSuchMethodException e) { - throw new IllegalArgumentException( - "Serializer " - + serializerClass.getName() - + " doesn't define a supported constructor for " - + type, - e); - } catch (Throwable t) { - ExceptionUtils.throwException(t); - throw new IllegalStateException("unreachable"); - } - } - - public static void write(WriteContext writeContext, Serializer serializer, T obj) { - serializer.write(writeContext, obj); - } - - public static T read(ReadContext readContext, Serializer serializer) { - return serializer.read(readContext); - } - - private static final ToIntFunction GET_CODER; - private static final Function GET_VALUE; - - static { - if (AndroidSupport.IS_ANDROID) { - GET_VALUE = null; - GET_CODER = null; - } else { - Function getValue; - ToIntFunction getCoder; - try { - getValue = (Function) makeGetterFunction(StringBuilder.class.getSuperclass(), "getValue"); - } catch (Throwable e) { - getValue = null; - } - try { - Method getCoderMethod = StringBuilder.class.getSuperclass().getDeclaredMethod("getCoder"); - getCoder = (ToIntFunction) makeGetterFunction(getCoderMethod, int.class); - } catch (NoSuchMethodException e) { - getCoder = null; - } catch (Throwable e) { - getCoder = null; - } - GET_VALUE = getValue; - GET_CODER = getCoder; - } - } - - public abstract static class AbstractStringBuilderSerializer - extends Serializer { - private final Config config; - - public AbstractStringBuilderSerializer(Config config, Class type) { - super(config, type); - this.config = config; - } - - @Override - public void write(WriteContext writeContext, T value) { - MemoryBuffer buffer = writeContext.getBuffer(); - StringSerializer stringSerializer = writeContext.getStringSerializer(); - if (config.isXlang()) { - stringSerializer.writeString(buffer, value.toString()); - return; - } - if (AndroidSupport.IS_ANDROID) { - stringSerializer.writeString(buffer, value.toString()); - return; - } - if (GET_VALUE == null) { - stringSerializer.writeString(buffer, value.toString()); - return; - } - if (GET_CODER != null) { - int coder = GET_CODER.applyAsInt(value); - byte[] v = (byte[]) GET_VALUE.apply(value); - int bytesLen = value.length(); - if (coder != 0) { - if (coder != 1) { - throw new UnsupportedOperationException("Unsupported coder " + coder); - } - bytesLen <<= 1; - } - long header = ((long) bytesLen << 2) | coder; - buffer.writeVarUInt64(header); - buffer.writeBytes(v, 0, bytesLen); - } else { - Object rawValue = GET_VALUE.apply(value); - if (!(rawValue instanceof char[])) { - stringSerializer.writeString(buffer, value.toString()); - return; - } - char[] v = (char[]) rawValue; - if (StringEncodingUtils.isLatin(v)) { - stringSerializer.writeCharsLatin1(buffer, v, value.length()); - } else { - stringSerializer.writeCharsUTF16(buffer, v, value.length()); - } - } - } - } - - public static final class StringBuilderSerializer - extends AbstractStringBuilderSerializer { - - public StringBuilderSerializer(Config config) { - super(config, StringBuilder.class); - } - - @Override - public StringBuilder copy(CopyContext copyContext, StringBuilder origin) { - return new StringBuilder(origin); - } - - @Override - public StringBuilder read(ReadContext readContext) { - return new StringBuilder(readContext.readString()); - } - } - - public static final class StringBufferSerializer - extends AbstractStringBuilderSerializer { - - public StringBufferSerializer(Config config) { - super(config, StringBuffer.class); - } - - @Override - public StringBuffer copy(CopyContext copyContext, StringBuffer origin) { - return new StringBuffer(origin); - } - - @Override - public StringBuffer read(ReadContext readContext) { - return new StringBuffer(readContext.readString()); - } - } - - public static final class StringTokenizerSerializer extends Serializer - implements Shareable { - public StringTokenizerSerializer(Config config) { - super(config, StringTokenizer.class); - } - - @Override - public void write(WriteContext writeContext, StringTokenizer value) { - checkStringTokenizerAccess(); - MemoryBuffer buffer = writeContext.getBuffer(); - writeContext.writeRef(Accessors.STR.getObject(value)); - writeContext.writeRef(Accessors.DELIMITERS.getObject(value)); - buffer.writeBoolean(Accessors.RET_DELIMS.getBoolean(value)); - buffer.writeVarInt32(Accessors.CURRENT_POSITION.getInt(value)); - buffer.writeVarInt32(Accessors.NEW_POSITION.getInt(value)); - buffer.writeBoolean(Accessors.DELIMS_CHANGED.getBoolean(value)); - } - - @Override - public StringTokenizer read(ReadContext readContext) { - checkStringTokenizerAccess(); - String str = (String) readContext.readRef(); - String delimiters = (String) readContext.readRef(); - boolean retDelims = readContext.getBuffer().readBoolean(); - StringTokenizer tokenizer = new StringTokenizer(str, delimiters, retDelims); - restoreState(readContext.getBuffer(), tokenizer); - return tokenizer; - } - - @Override - public StringTokenizer copy(CopyContext copyContext, StringTokenizer value) { - checkStringTokenizerAccess(); - StringTokenizer tokenizer = - new StringTokenizer( - (String) Accessors.STR.getObject(value), - (String) Accessors.DELIMITERS.getObject(value), - Accessors.RET_DELIMS.getBoolean(value)); - Accessors.CURRENT_POSITION.putInt(tokenizer, Accessors.CURRENT_POSITION.getInt(value)); - Accessors.NEW_POSITION.putInt(tokenizer, Accessors.NEW_POSITION.getInt(value)); - Accessors.DELIMS_CHANGED.putBoolean(tokenizer, Accessors.DELIMS_CHANGED.getBoolean(value)); - return tokenizer; - } - - private static void restoreState(MemoryBuffer buffer, StringTokenizer tokenizer) { - Accessors.CURRENT_POSITION.putInt(tokenizer, buffer.readVarInt32()); - Accessors.NEW_POSITION.putInt(tokenizer, buffer.readVarInt32()); - Accessors.DELIMS_CHANGED.putBoolean(tokenizer, buffer.readBoolean()); - } - - private static void checkStringTokenizerAccess() { - if (!MemoryUtils.JDK_COLLECTION_FIELD_ACCESS) { - throw stringTokenizerAccessError(); - } - } - - private static UnsupportedOperationException stringTokenizerAccessError() { - return new UnsupportedOperationException( - "StringTokenizer serialization requires JDK internal field access. On JDK25+, open " - + "java.base/java.util to org.apache.fory.core,org.apache.fory.format."); - } - - private static final class Accessors { - private static final FieldAccessor CURRENT_POSITION = - FieldAccessor.createAccessor( - ReflectionUtils.getField(StringTokenizer.class, "currentPosition")); - private static final FieldAccessor NEW_POSITION = - FieldAccessor.createAccessor( - ReflectionUtils.getField(StringTokenizer.class, "newPosition")); - private static final FieldAccessor STR = - FieldAccessor.createAccessor(ReflectionUtils.getField(StringTokenizer.class, "str")); - private static final FieldAccessor DELIMITERS = - FieldAccessor.createAccessor( - ReflectionUtils.getField(StringTokenizer.class, "delimiters")); - private static final FieldAccessor RET_DELIMS = - FieldAccessor.createAccessor( - ReflectionUtils.getField(StringTokenizer.class, "retDelims")); - private static final FieldAccessor DELIMS_CHANGED = - FieldAccessor.createAccessor( - ReflectionUtils.getField(StringTokenizer.class, "delimsChanged")); - } - } - - public static final class AtomicBooleanSerializer extends Serializer - implements Shareable { - - public AtomicBooleanSerializer(Config config) { - super(config, AtomicBoolean.class); - } - - @Override - public void write(WriteContext writeContext, AtomicBoolean value) { - writeContext.getBuffer().writeBoolean(value.get()); - } - - @Override - public AtomicBoolean copy(CopyContext copyContext, AtomicBoolean origin) { - return new AtomicBoolean(origin.get()); - } - - @Override - public AtomicBoolean read(ReadContext readContext) { - return new AtomicBoolean(readContext.getBuffer().readBoolean()); - } - } - - public static final class AtomicIntegerSerializer extends Serializer - implements Shareable { - - public AtomicIntegerSerializer(Config config) { - super(config, AtomicInteger.class); - } - - @Override - public void write(WriteContext writeContext, AtomicInteger value) { - writeContext.getBuffer().writeInt32(value.get()); - } - - @Override - public AtomicInteger copy(CopyContext copyContext, AtomicInteger origin) { - return new AtomicInteger(origin.get()); - } - - @Override - public AtomicInteger read(ReadContext readContext) { - return new AtomicInteger(readContext.getBuffer().readInt32()); - } - } - - public static final class AtomicLongSerializer extends Serializer - implements Shareable { - - public AtomicLongSerializer(Config config) { - super(config, AtomicLong.class); - } - - @Override - public void write(WriteContext writeContext, AtomicLong value) { - writeContext.getBuffer().writeInt64(value.get()); - } - - @Override - public AtomicLong copy(CopyContext copyContext, AtomicLong origin) { - return new AtomicLong(origin.get()); - } - - @Override - public AtomicLong read(ReadContext readContext) { - return new AtomicLong(readContext.getBuffer().readInt64()); - } - } - - public static final class AtomicReferenceSerializer extends Serializer - implements Shareable { - - public AtomicReferenceSerializer(Config config) { - super(config, AtomicReference.class); - } - - @Override - public void write(WriteContext writeContext, AtomicReference value) { - writeContext.writeRef(value.get()); - } - - @Override - public AtomicReference copy(CopyContext copyContext, AtomicReference origin) { - return new AtomicReference(copyContext.copyObject(origin.get())); - } - - @Override - public AtomicReference read(ReadContext readContext) { - return new AtomicReference(readContext.readRef()); - } - } - - public static final class CurrencySerializer extends ImmutableSerializer - implements Shareable { - public CurrencySerializer(Config config) { - super(config, Currency.class); - } - - @Override - public void write(WriteContext writeContext, Currency object) { - writeContext.writeString(object.getCurrencyCode()); - } - - @Override - public Currency read(ReadContext readContext) { - return Currency.getInstance(readContext.readString()); - } - } - - /** Serializer for {@link Charset}. */ - public static final class CharsetSerializer extends ImmutableSerializer - implements Shareable { - public CharsetSerializer(Config config, Class type) { - super(config, type); - } - - public void write(WriteContext writeContext, T object) { - writeContext.writeString(object.name()); - } - - public T read(ReadContext readContext) { - return (T) Charset.forName(readContext.readString()); - } - } - - public static final class URISerializer extends ImmutableSerializer - implements Shareable { - - public URISerializer(Config config) { - super(config, URI.class); - } - - @Override - public void write(WriteContext writeContext, final URI uri) { - writeContext.writeString(uri.toString()); - } - - @Override - public URI read(ReadContext readContext) { - return URI.create(readContext.readString()); - } - } - - public static final class RegexSerializer extends ImmutableSerializer - implements Shareable { - public RegexSerializer(Config config) { - super(config, Pattern.class); - } - - @Override - public void write(WriteContext writeContext, Pattern pattern) { - MemoryBuffer buffer = writeContext.getBuffer(); - writeContext.writeString(pattern.pattern()); - buffer.writeInt32(pattern.flags()); - } - - @Override - public Pattern read(ReadContext readContext) { - MemoryBuffer buffer = readContext.getBuffer(); - String regex = readContext.readString(); - int flags = buffer.readInt32(); - return Pattern.compile(regex, flags); - } - } - - public static final class UUIDSerializer extends ImmutableSerializer implements Shareable { - - public UUIDSerializer(Config config) { - super(config, UUID.class); - } - - @Override - public void write(WriteContext writeContext, final UUID uuid) { - MemoryBuffer buffer = writeContext.getBuffer(); - buffer.writeInt64(uuid.getMostSignificantBits()); - buffer.writeInt64(uuid.getLeastSignificantBits()); - } - - @Override - public UUID read(ReadContext readContext) { - MemoryBuffer buffer = readContext.getBuffer(); - return new UUID(buffer.readInt64(), buffer.readInt64()); - } - } - - public static final class ClassSerializer extends ImmutableSerializer - implements Shareable { - public ClassSerializer(Config config) { - super(config, Class.class); - } - - @Override - public void write(WriteContext writeContext, Class value) { - ((ClassResolver) writeContext.getTypeResolver()).writeClassInternal(writeContext, value); - } - - @Override - public Class read(ReadContext readContext) { - return ((ClassResolver) readContext.getTypeResolver()).readClassInternal(readContext); - } - } - - /** - * Serializer for empty object of type {@link Object}. Fory disabled serialization for jdk - * internal types which doesn't implement {@link java.io.Serializable} for security, but empty - * object is safe and used sometimes, so fory should support its serialization without disable - * serializable or class registration checks. - */ - // Use a separate serializer to avoid codegen for empty object. - public static final class EmptyObjectSerializer extends ImmutableSerializer - implements Shareable { - - public EmptyObjectSerializer(Config config) { - super(config, Object.class); - } - - @Override - public void write(WriteContext writeContext, Object value) {} - - @Override - public Object read(ReadContext readContext) { - return new Object(); - } - } - - public static void registerDefaultSerializers(TypeResolver resolver) { - Config config = resolver.getConfig(); - resolver.registerInternalSerializer(Class.class, new ClassSerializer(config)); - resolver.registerInternalSerializer(StringBuilder.class, new StringBuilderSerializer(config)); - resolver.registerInternalSerializer(StringBuffer.class, new StringBufferSerializer(config)); - // Keep this internal type id reserved even when JDK collection internals are not open; - // otherwise payloads written with access enabled decode later collection ids incorrectly. - resolver.registerInternalSerializer(StringTokenizer.class, new StringTokenizerSerializer(config)); - resolver.registerInternalSerializer(BigInteger.class, new BigIntegerSerializer(config)); - resolver.registerInternalSerializer(BigDecimal.class, new DecimalSerializer(config)); - resolver.registerInternalSerializer(AtomicBoolean.class, new AtomicBooleanSerializer(config)); - resolver.registerInternalSerializer(AtomicInteger.class, new AtomicIntegerSerializer(config)); - resolver.registerInternalSerializer(AtomicLong.class, new AtomicLongSerializer(config)); - resolver.registerInternalSerializer( - AtomicReference.class, new AtomicReferenceSerializer(config)); - resolver.registerInternalSerializer(Currency.class, new CurrencySerializer(config)); - resolver.registerInternalSerializer(URI.class, new URISerializer(config)); - resolver.registerInternalSerializer(Pattern.class, new RegexSerializer(config)); - resolver.registerInternalSerializer(UUID.class, new UUIDSerializer(config)); - resolver.registerInternalSerializer(Object.class, new EmptyObjectSerializer(config)); - } -} From 1daa83ec1ed3a293c4a666e6ad1434fc9a2e643c Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sun, 24 May 2026 18:00:33 +0800 Subject: [PATCH 28/69] docs(java): update final field mutation guidance --- docs/guide/java/troubleshooting.md | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/docs/guide/java/troubleshooting.md b/docs/guide/java/troubleshooting.md index f4837e211c..a28518ffa7 100644 --- a/docs/guide/java/troubleshooting.md +++ b/docs/guide/java/troubleshooting.md @@ -183,10 +183,15 @@ For example, direct `ByteBuffer` wrapping on the module path requires: --add-opens=java.base/java.nio=ALL-UNNAMED,org.apache.fory.core,org.apache.fory.format ``` -Normal classes with final instance fields need a constructor that covers those final fields when -Unsafe allocation is denied. Annotate the constructor with -`java.beans.ConstructorProperties`, or compile the class with `-parameters` so Fory can bind -constructor parameters to fields. Non-final fields can still be restored after construction. +Normal classes with final instance fields require final-field mutation to be enabled for Fory core +when Unsafe allocation is denied: + +```bash +--enable-final-field-mutation=org.apache.fory.core +``` + +Fory restores those final fields through method-handle based access. Non-final fields can still be +restored through generated direct field assignment where available. The vectorized Arrow APIs in `fory-format` depend on Apache Arrow's memory layer. With the current Arrow dependency, those APIs are unavailable when `--sun-misc-unsafe-memory-access=deny` is set From 7272e5db25ee8c71be07c5f7422a7cfeea2f1082 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sun, 24 May 2026 18:30:05 +0800 Subject: [PATCH 29/69] fix(java): restore final fields with method handles on JDK25 --- .../apache/fory/reflect/ObjectCreators.java | 64 +---- .../apache/fory/reflect/FieldAccessor.java | 240 +++++++++++++----- .../fory/reflect/FieldAccessorTest.java | 24 ++ .../fory/serializer/ObjectSerializerTest.java | 109 ++++++++ 4 files changed, 310 insertions(+), 127 deletions(-) diff --git a/java/fory-core/src/main/java/org/apache/fory/reflect/ObjectCreators.java b/java/fory-core/src/main/java/org/apache/fory/reflect/ObjectCreators.java index a9aa8d2da9..682a9546be 100644 --- a/java/fory-core/src/main/java/org/apache/fory/reflect/ObjectCreators.java +++ b/java/fory-core/src/main/java/org/apache/fory/reflect/ObjectCreators.java @@ -28,7 +28,6 @@ import java.lang.reflect.Modifier; import java.lang.reflect.Parameter; import java.util.ArrayList; -import java.util.Arrays; import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; @@ -110,9 +109,6 @@ private static ObjectCreator creategetObjectCreator(Class type) { if (JdkVersion.MAJOR_VERSION >= 25 && noArgConstructor == null) { return new ConstructorObjectCreator<>(type); } - if (JdkVersion.MAJOR_VERSION >= 25 && hasFinalFields(type)) { - return new ConstructorObjectCreator<>(type); - } if (GraalvmSupport.IN_GRAALVM_NATIVE_IMAGE) { if (noArgConstructor != null) { return new DeclaredNoArgCtrObjectCreator<>(type); @@ -126,15 +122,6 @@ private static ObjectCreator creategetObjectCreator(Class type) { return new DeclaredNoArgCtrObjectCreator<>(type); } - private static boolean hasFinalFields(Class type) { - for (Field field : Descriptor.getFields(type)) { - if (Modifier.isFinal(field.getModifiers())) { - return true; - } - } - return false; - } - public static boolean supportsJdk25Creation(Class type) { if (JdkVersion.MAJOR_VERSION < 25 || RecordUtils.isRecord(type)) { return true; @@ -179,7 +166,6 @@ private static ConstructorMatch findConstructor(Class type) { Map fieldsById = new LinkedHashMap<>(); Set duplicateNames = new LinkedHashSet<>(); Set duplicateIds = new LinkedHashSet<>(); - Set finalFields = new LinkedHashSet<>(); for (Field field : fields) { fieldsByNameList.computeIfAbsent(field.getName(), name -> new ArrayList<>()).add(field); Field previous = fieldsByName.put(field.getName(), field); @@ -193,9 +179,6 @@ private static ConstructorMatch findConstructor(Class type) { duplicateIds.add(foryField.id()); } } - if (Modifier.isFinal(field.getModifiers())) { - finalFields.add(field); - } } ConstructorMatch best = null; for (Constructor constructor : type.getDeclaredConstructors()) { @@ -210,20 +193,15 @@ private static ConstructorMatch findConstructor(Class type) { fieldsByNameList, fieldsById, duplicateNames, - duplicateIds, - finalFields); + duplicateIds); if (match != null && (best == null || match.score > best.score)) { best = match; } } if (best == null) { - String requirement = - finalFields.isEmpty() - ? "a bindable constructor because no no-arg constructor is available" - : "a constructor covering final fields " + finalFields; throw new ForyException( "JDK25 zero-Unsafe mode requires " - + requirement + + "a bindable constructor because no no-arg constructor is available" + " for " + type + ". Annotate the constructor with java.beans.ConstructorProperties or compile " @@ -239,22 +217,18 @@ private static ConstructorMatch matchConstructor( Map> fieldsByNameList, Map fieldsById, Set duplicateNames, - Set duplicateIds, - Set finalFields) { + Set duplicateIds) { Field[] fields = constructorFields( constructor, fieldsByName, fieldsByNameList, fieldsById, duplicateNames, duplicateIds); if (fields == null) { return null; } - return matchConstructorFields(constructor, finalFields, fields); + return matchConstructorFields(constructor, fields); } private static ConstructorMatch matchConstructorFields( - Constructor constructor, Set finalFields, Field[] fields) { - if (!containsAllFinalFields(finalFields, fields)) { - return null; - } + Constructor constructor, Field[] fields) { Class[] parameterTypes = constructor.getParameterTypes(); String[] names = new String[fields.length]; Class[] declaringClasses = new Class[fields.length]; @@ -274,34 +248,6 @@ private static ConstructorMatch matchConstructorFields( constructor, names, declaringClasses, fieldTypes, finalFieldFlags, 300 - fields.length); } - private static boolean containsAllFinalFields(Set finalFields, Field[] fields) { - Set selectedFields = new LinkedHashSet<>(Arrays.asList(fields)); - for (Field finalField : finalFields) { - if (selectedFields.contains(finalField)) { - continue; - } - if (coveredBySyntheticField(finalField, selectedFields)) { - continue; - } - return false; - } - return true; - } - - private static boolean coveredBySyntheticField(Field finalField, Set selectedFields) { - if (!finalField.isSynthetic()) { - return false; - } - for (Field selectedField : selectedFields) { - if (selectedField.isSynthetic() - && selectedField.getName().equals(finalField.getName()) - && selectedField.getType() == finalField.getType()) { - return true; - } - } - return false; - } - private static Field[] constructorFields( Constructor constructor, Map fieldsByName, diff --git a/java/fory-core/src/main/java25/org/apache/fory/reflect/FieldAccessor.java b/java/fory-core/src/main/java25/org/apache/fory/reflect/FieldAccessor.java index 82c8cc21e9..1c06e64634 100644 --- a/java/fory-core/src/main/java25/org/apache/fory/reflect/FieldAccessor.java +++ b/java/fory-core/src/main/java25/org/apache/fory/reflect/FieldAccessor.java @@ -407,6 +407,16 @@ private static UnsupportedOperationException unsupportedWrite(Field field, Throw "Field cannot be written through supported JDK access APIs: " + field, cause); } + private static IllegalStateException finalMutationFailure(Field field, Throwable cause) { + return new IllegalStateException( + "Cannot write final field " + + field + + ". On JDK25+, start the JVM with " + + "--enable-final-field-mutation=org.apache.fory.core and open the declaring " + + "package to org.apache.fory.core,org.apache.fory.format.", + cause); + } + private static RuntimeException getterFailure(Field field, Throwable cause) { return new RuntimeException("Failed to read record field: " + field, cause); } @@ -450,102 +460,124 @@ public void set(Object obj, Object value) { private abstract static class VarHandleAccessor extends FieldAccessor { protected final VarHandle handle; protected final boolean isStatic; + protected final boolean isFinal; + protected volatile MethodHandle finalSetter; VarHandleAccessor(Field field) { super(field, -1); handle = fieldHandle(field); isStatic = Modifier.isStatic(field.getModifiers()); + isFinal = Modifier.isFinal(field.getModifiers()); } - protected void setReflectively(Object obj, Object value, Throwable cause) { + private static MethodHandle createFinalSetter(Field field) { try { - prepareReflectiveWrite(cause); - field.set(target(obj), value); + field.setAccessible(true); + return privateLookup(field).unreflectSetter(field); } catch (IllegalAccessException | RuntimeException e) { - throw unsupportedWrite(field, e); + throw finalMutationFailure(field, e); } } - private Object target(Object obj) { - return isStatic ? null : obj; + private MethodHandle finalSetter(Throwable cause) { + if (isStatic || !isFinal) { + throw unsupportedWrite(field, cause); + } + MethodHandle setter = finalSetter; + if (setter == null) { + setter = createFinalSetter(field); + finalSetter = setter; + } + return setter; } - private void prepareReflectiveWrite(Throwable cause) { - if (field.getDeclaringClass().getName().startsWith("java.")) { - throw unsupportedWrite(field, cause); + protected void setFinal(Object obj, Object value, Throwable cause) { + MethodHandle setter = finalSetter(cause); + checkObj(obj); + try { + setter.invoke(obj, value); + } catch (Throwable e) { + throw finalMutationFailure(field, e); } - field.setAccessible(true); } - protected void setBooleanReflectively(Object obj, boolean value, Throwable cause) { + protected void setFinalBoolean(Object obj, boolean value, Throwable cause) { + MethodHandle setter = finalSetter(cause); + checkObj(obj); try { - prepareReflectiveWrite(cause); - field.setBoolean(target(obj), value); - } catch (IllegalAccessException | RuntimeException e) { - throw unsupportedWrite(field, e); + setter.invoke(obj, value); + } catch (Throwable e) { + throw finalMutationFailure(field, e); } } - protected void setByteReflectively(Object obj, byte value, Throwable cause) { + protected void setFinalByte(Object obj, byte value, Throwable cause) { + MethodHandle setter = finalSetter(cause); + checkObj(obj); try { - prepareReflectiveWrite(cause); - field.setByte(target(obj), value); - } catch (IllegalAccessException | RuntimeException e) { - throw unsupportedWrite(field, e); + setter.invoke(obj, value); + } catch (Throwable e) { + throw finalMutationFailure(field, e); } } - protected void setCharReflectively(Object obj, char value, Throwable cause) { + protected void setFinalChar(Object obj, char value, Throwable cause) { + MethodHandle setter = finalSetter(cause); + checkObj(obj); try { - prepareReflectiveWrite(cause); - field.setChar(target(obj), value); - } catch (IllegalAccessException | RuntimeException e) { - throw unsupportedWrite(field, e); + setter.invoke(obj, value); + } catch (Throwable e) { + throw finalMutationFailure(field, e); } } - protected void setShortReflectively(Object obj, short value, Throwable cause) { + protected void setFinalShort(Object obj, short value, Throwable cause) { + MethodHandle setter = finalSetter(cause); + checkObj(obj); try { - prepareReflectiveWrite(cause); - field.setShort(target(obj), value); - } catch (IllegalAccessException | RuntimeException e) { - throw unsupportedWrite(field, e); + setter.invoke(obj, value); + } catch (Throwable e) { + throw finalMutationFailure(field, e); } } - protected void setIntReflectively(Object obj, int value, Throwable cause) { + protected void setFinalInt(Object obj, int value, Throwable cause) { + MethodHandle setter = finalSetter(cause); + checkObj(obj); try { - prepareReflectiveWrite(cause); - field.setInt(target(obj), value); - } catch (IllegalAccessException | RuntimeException e) { - throw unsupportedWrite(field, e); + setter.invoke(obj, value); + } catch (Throwable e) { + throw finalMutationFailure(field, e); } } - protected void setLongReflectively(Object obj, long value, Throwable cause) { + protected void setFinalLong(Object obj, long value, Throwable cause) { + MethodHandle setter = finalSetter(cause); + checkObj(obj); try { - prepareReflectiveWrite(cause); - field.setLong(target(obj), value); - } catch (IllegalAccessException | RuntimeException e) { - throw unsupportedWrite(field, e); + setter.invoke(obj, value); + } catch (Throwable e) { + throw finalMutationFailure(field, e); } } - protected void setFloatReflectively(Object obj, float value, Throwable cause) { + protected void setFinalFloat(Object obj, float value, Throwable cause) { + MethodHandle setter = finalSetter(cause); + checkObj(obj); try { - prepareReflectiveWrite(cause); - field.setFloat(target(obj), value); - } catch (IllegalAccessException | RuntimeException e) { - throw unsupportedWrite(field, e); + setter.invoke(obj, value); + } catch (Throwable e) { + throw finalMutationFailure(field, e); } } - protected void setDoubleReflectively(Object obj, double value, Throwable cause) { + protected void setFinalDouble(Object obj, double value, Throwable cause) { + MethodHandle setter = finalSetter(cause); + checkObj(obj); try { - prepareReflectiveWrite(cause); - field.setDouble(target(obj), value); - } catch (IllegalAccessException | RuntimeException e) { - throw unsupportedWrite(field, e); + setter.invoke(obj, value); + } catch (Throwable e) { + throw finalMutationFailure(field, e); } } } @@ -578,6 +610,10 @@ public void set(Object obj, Object value) { @Override public void putBoolean(Object obj, boolean value) { + if (isFinal) { + setFinalBoolean(obj, value, null); + return; + } try { if (isStatic) { handle.set(value); @@ -586,7 +622,7 @@ public void putBoolean(Object obj, boolean value) { handle.set(obj, value); } } catch (UnsupportedOperationException e) { - setBooleanReflectively(obj, value, e); + setFinalBoolean(obj, value, e); } } } @@ -656,6 +692,10 @@ public void set(Object obj, Object value) { @Override public void putByte(Object obj, byte value) { + if (isFinal) { + setFinalByte(obj, value, null); + return; + } try { if (isStatic) { handle.set(value); @@ -664,7 +704,7 @@ public void putByte(Object obj, byte value) { handle.set(obj, value); } } catch (UnsupportedOperationException e) { - setByteReflectively(obj, value, e); + setFinalByte(obj, value, e); } } } @@ -735,6 +775,10 @@ public void set(Object obj, Object value) { @Override public void putChar(Object obj, char value) { + if (isFinal) { + setFinalChar(obj, value, null); + return; + } try { if (isStatic) { handle.set(value); @@ -743,7 +787,7 @@ public void putChar(Object obj, char value) { handle.set(obj, value); } } catch (UnsupportedOperationException e) { - setCharReflectively(obj, value, e); + setFinalChar(obj, value, e); } } } @@ -813,6 +857,10 @@ public void set(Object obj, Object value) { @Override public void putShort(Object obj, short value) { + if (isFinal) { + setFinalShort(obj, value, null); + return; + } try { if (isStatic) { handle.set(value); @@ -821,7 +869,7 @@ public void putShort(Object obj, short value) { handle.set(obj, value); } } catch (UnsupportedOperationException e) { - setShortReflectively(obj, value, e); + setFinalShort(obj, value, e); } } } @@ -891,6 +939,10 @@ public void set(Object obj, Object value) { @Override public void putInt(Object obj, int value) { + if (isFinal) { + setFinalInt(obj, value, null); + return; + } try { if (isStatic) { handle.set(value); @@ -899,7 +951,7 @@ public void putInt(Object obj, int value) { handle.set(obj, value); } } catch (UnsupportedOperationException e) { - setIntReflectively(obj, value, e); + setFinalInt(obj, value, e); } } } @@ -969,6 +1021,10 @@ public void set(Object obj, Object value) { @Override public void putLong(Object obj, long value) { + if (isFinal) { + setFinalLong(obj, value, null); + return; + } try { if (isStatic) { handle.set(value); @@ -977,7 +1033,7 @@ public void putLong(Object obj, long value) { handle.set(obj, value); } } catch (UnsupportedOperationException e) { - setLongReflectively(obj, value, e); + setFinalLong(obj, value, e); } } } @@ -1047,6 +1103,10 @@ public void set(Object obj, Object value) { @Override public void putFloat(Object obj, float value) { + if (isFinal) { + setFinalFloat(obj, value, null); + return; + } try { if (isStatic) { handle.set(value); @@ -1055,7 +1115,7 @@ public void putFloat(Object obj, float value) { handle.set(obj, value); } } catch (UnsupportedOperationException e) { - setFloatReflectively(obj, value, e); + setFinalFloat(obj, value, e); } } } @@ -1125,6 +1185,10 @@ public void set(Object obj, Object value) { @Override public void putDouble(Object obj, double value) { + if (isFinal) { + setFinalDouble(obj, value, null); + return; + } try { if (isStatic) { handle.set(value); @@ -1133,7 +1197,7 @@ public void putDouble(Object obj, double value) { handle.set(obj, value); } } catch (UnsupportedOperationException e) { - setDoubleReflectively(obj, value, e); + setFinalDouble(obj, value, e); } } } @@ -1193,6 +1257,10 @@ public Object get(Object obj) { @Override public void set(Object obj, Object value) { + if (isFinal) { + setFinal(obj, value, null); + return; + } try { if (isStatic) { handle.set(value); @@ -1201,7 +1269,7 @@ public void set(Object obj, Object value) { handle.set(obj, value); } } catch (UnsupportedOperationException e) { - setReflectively(obj, value, e); + setFinal(obj, value, e); } } } @@ -1289,6 +1357,10 @@ public Object get(Object obj) { @Override public void set(Object obj, Object value) { + if (isFinal) { + setFinal(obj, value, null); + return; + } try { if (isStatic) { handle.set(value); @@ -1297,7 +1369,7 @@ public void set(Object obj, Object value) { handle.set(obj, value); } } catch (UnsupportedOperationException e) { - setReflectively(obj, value, e); + setFinal(obj, value, e); } catch (RuntimeException e) { throw accessorFailure(field, e); } @@ -1318,6 +1390,10 @@ public boolean getBoolean(Object obj) { @Override public void putBoolean(Object obj, boolean value) { + if (isFinal) { + setFinalBoolean(obj, value, null); + return; + } try { if (isStatic) { handle.set(value); @@ -1326,7 +1402,7 @@ public void putBoolean(Object obj, boolean value) { handle.set(obj, value); } } catch (UnsupportedOperationException e) { - setBooleanReflectively(obj, value, e); + setFinalBoolean(obj, value, e); } catch (RuntimeException e) { throw accessorFailure(field, e); } @@ -1347,6 +1423,10 @@ public byte getByte(Object obj) { @Override public void putByte(Object obj, byte value) { + if (isFinal) { + setFinalByte(obj, value, null); + return; + } try { if (isStatic) { handle.set(value); @@ -1355,7 +1435,7 @@ public void putByte(Object obj, byte value) { handle.set(obj, value); } } catch (UnsupportedOperationException e) { - setByteReflectively(obj, value, e); + setFinalByte(obj, value, e); } catch (RuntimeException e) { throw accessorFailure(field, e); } @@ -1376,6 +1456,10 @@ public char getChar(Object obj) { @Override public void putChar(Object obj, char value) { + if (isFinal) { + setFinalChar(obj, value, null); + return; + } try { if (isStatic) { handle.set(value); @@ -1384,7 +1468,7 @@ public void putChar(Object obj, char value) { handle.set(obj, value); } } catch (UnsupportedOperationException e) { - setCharReflectively(obj, value, e); + setFinalChar(obj, value, e); } catch (RuntimeException e) { throw accessorFailure(field, e); } @@ -1405,6 +1489,10 @@ public short getShort(Object obj) { @Override public void putShort(Object obj, short value) { + if (isFinal) { + setFinalShort(obj, value, null); + return; + } try { if (isStatic) { handle.set(value); @@ -1413,7 +1501,7 @@ public void putShort(Object obj, short value) { handle.set(obj, value); } } catch (UnsupportedOperationException e) { - setShortReflectively(obj, value, e); + setFinalShort(obj, value, e); } catch (RuntimeException e) { throw accessorFailure(field, e); } @@ -1434,6 +1522,10 @@ public int getInt(Object obj) { @Override public void putInt(Object obj, int value) { + if (isFinal) { + setFinalInt(obj, value, null); + return; + } try { if (isStatic) { handle.set(value); @@ -1442,7 +1534,7 @@ public void putInt(Object obj, int value) { handle.set(obj, value); } } catch (UnsupportedOperationException e) { - setIntReflectively(obj, value, e); + setFinalInt(obj, value, e); } catch (RuntimeException e) { throw accessorFailure(field, e); } @@ -1463,6 +1555,10 @@ public long getLong(Object obj) { @Override public void putLong(Object obj, long value) { + if (isFinal) { + setFinalLong(obj, value, null); + return; + } try { if (isStatic) { handle.set(value); @@ -1471,7 +1567,7 @@ public void putLong(Object obj, long value) { handle.set(obj, value); } } catch (UnsupportedOperationException e) { - setLongReflectively(obj, value, e); + setFinalLong(obj, value, e); } catch (RuntimeException e) { throw accessorFailure(field, e); } @@ -1492,6 +1588,10 @@ public float getFloat(Object obj) { @Override public void putFloat(Object obj, float value) { + if (isFinal) { + setFinalFloat(obj, value, null); + return; + } try { if (isStatic) { handle.set(value); @@ -1500,7 +1600,7 @@ public void putFloat(Object obj, float value) { handle.set(obj, value); } } catch (UnsupportedOperationException e) { - setFloatReflectively(obj, value, e); + setFinalFloat(obj, value, e); } catch (RuntimeException e) { throw accessorFailure(field, e); } @@ -1521,6 +1621,10 @@ public double getDouble(Object obj) { @Override public void putDouble(Object obj, double value) { + if (isFinal) { + setFinalDouble(obj, value, null); + return; + } try { if (isStatic) { handle.set(value); @@ -1529,7 +1633,7 @@ public void putDouble(Object obj, double value) { handle.set(obj, value); } } catch (UnsupportedOperationException e) { - setDoubleReflectively(obj, value, e); + setFinalDouble(obj, value, e); } catch (RuntimeException e) { throw accessorFailure(field, e); } diff --git a/java/fory-core/src/test/java/org/apache/fory/reflect/FieldAccessorTest.java b/java/fory-core/src/test/java/org/apache/fory/reflect/FieldAccessorTest.java index c86faa6bf5..0b333ec972 100644 --- a/java/fory-core/src/test/java/org/apache/fory/reflect/FieldAccessorTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/reflect/FieldAccessorTest.java @@ -90,6 +90,20 @@ public void testHiddenAccessor() throws Exception { } } + @Test + public void testFinalFieldWrites() throws Exception { + FinalFields fields = new FinalFields(1, "a"); + FieldAccessor intAccessor = + FieldAccessor.createAccessor(FinalFields.class.getDeclaredField("intValue")); + intAccessor.putInt(fields, 2); + Assert.assertEquals(intAccessor.getInt(fields), 2); + + FieldAccessor objectAccessor = + FieldAccessor.createAccessor(FinalFields.class.getDeclaredField("objectValue")); + objectAccessor.putObject(fields, "b"); + Assert.assertEquals(objectAccessor.getObject(fields), "b"); + } + private static boolean isHidden(Class cls) throws Exception { return (Boolean) Class.class.getMethod("isHidden").invoke(cls); } @@ -180,4 +194,14 @@ private static final class HiddenFields { private String text = "a"; private final long finalValue = 3; } + + private static final class FinalFields { + private final int intValue; + private final Object objectValue; + + private FinalFields(int value, Object object) { + intValue = value; + objectValue = object; + } + } } diff --git a/java/fory-core/src/test/java/org/apache/fory/serializer/ObjectSerializerTest.java b/java/fory-core/src/test/java/org/apache/fory/serializer/ObjectSerializerTest.java index 8eb70dffa5..178a1cac9e 100644 --- a/java/fory-core/src/test/java/org/apache/fory/serializer/ObjectSerializerTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/serializer/ObjectSerializerTest.java @@ -239,6 +239,115 @@ public static final class ConstructorBackrefChild { private ConstructorBackrefRoot root; } + public static final class FinalNoArgBean { + private final int id; + private final String name; + private int count; + + public FinalNoArgBean() { + id = -1; + name = "default"; + } + + private FinalNoArgBean(int value, String text, int total) { + id = value; + name = text; + count = total; + } + } + + public static final class FinalPostCtorBean { + private final int id; + private String label; + + @ConstructorProperties("label") + public FinalPostCtorBean(String label) { + id = -1; + this.label = label; + } + + private FinalPostCtorBean(int value, String label) { + id = value; + this.label = label; + } + } + + @Test + public void testFinalNoArgRestore() { + FinalNoArgBean value = new FinalNoArgBean(7, "source", 9); + for (boolean codegen : new boolean[] {false, true}) { + Fory fory = + Fory.builder() + .withXlang(false) + .withRefTracking(true) + .withCodegen(codegen) + .requireClassRegistration(false) + .build(); + FinalNoArgBean newValue = (FinalNoArgBean) fory.deserialize(fory.serialize(value)); + assertEquals(newValue.id, value.id); + assertEquals(newValue.name, value.name); + assertEquals(newValue.count, value.count); + } + } + + @Test + public void testFinalNoArgRestoreCodegen() { + FinalNoArgBean value = new FinalNoArgBean(7, "source", 9); + Fory fory = + Fory.builder() + .withXlang(false) + .withRefTracking(true) + .withCodegen(true) + .requireClassRegistration(false) + .build(); + Serializer serializer = + Serializers.newSerializer( + fory, + FinalNoArgBean.class, + CodecUtils.loadOrGenObjectCodecClass(FinalNoArgBean.class, fory)); + FinalNoArgBean newValue = roundTripWithSerializer(fory, serializer, value); + assertEquals(newValue.id, value.id); + assertEquals(newValue.name, value.name); + assertEquals(newValue.count, value.count); + } + + @Test + public void testFinalPostCtorRestore() { + FinalPostCtorBean value = new FinalPostCtorBean(8, "ctor"); + for (boolean codegen : new boolean[] {false, true}) { + Fory fory = + Fory.builder() + .withXlang(false) + .withRefTracking(true) + .withCodegen(codegen) + .requireClassRegistration(false) + .build(); + FinalPostCtorBean newValue = (FinalPostCtorBean) fory.deserialize(fory.serialize(value)); + assertEquals(newValue.id, value.id); + assertEquals(newValue.label, value.label); + } + } + + @Test + public void testFinalPostCtorCodegen() { + FinalPostCtorBean value = new FinalPostCtorBean(8, "ctor"); + Fory fory = + Fory.builder() + .withXlang(false) + .withRefTracking(true) + .withCodegen(true) + .requireClassRegistration(false) + .build(); + Serializer serializer = + Serializers.newSerializer( + fory, + FinalPostCtorBean.class, + CodecUtils.loadOrGenObjectCodecClass(FinalPostCtorBean.class, fory)); + FinalPostCtorBean newValue = roundTripWithSerializer(fory, serializer, value); + assertEquals(newValue.id, value.id); + assertEquals(newValue.label, value.label); + } + @Test public void testConstructorFieldProtocolOrder() { ConstructorOrder value = new ConstructorOrder("root"); From d5098bc0a631b323be521daec3c98aac7ad858df Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sun, 24 May 2026 19:06:25 +0800 Subject: [PATCH 30/69] refactor(java): fold JDK25 codec builders into root --- .../android/AndroidForyRuntimeScenarios.java | 8 +- java/fory-core/pom.xml | 14 - .../org/apache/fory/builder/CodecBuilder.java | 69 +- .../fory/builder/ObjectCodecBuilder.java | 602 ++++++- .../org/apache/fory/memory/MemoryBuffer.java | 134 +- .../org/apache/fory/memory/MemoryOps.java | 102 ++ .../org/apache/fory/builder/CodecBuilder.java | 732 -------- .../fory/builder/ObjectCodecBuilder.java | 1465 ----------------- .../org/apache/fory/memory/MemoryBuffer.java | 101 +- .../apache/fory/memory/MemoryBufferTest.java | 69 +- .../fory/format/row/binary/BinaryArray.java | 14 +- .../fory/format/row/binary/BinaryMap.java | 7 +- 12 files changed, 1061 insertions(+), 2256 deletions(-) delete mode 100644 java/fory-core/src/main/java25/org/apache/fory/builder/CodecBuilder.java delete mode 100644 java/fory-core/src/main/java25/org/apache/fory/builder/ObjectCodecBuilder.java diff --git a/integration_tests/android_tests/src/main/java/org/apache/fory/android/AndroidForyRuntimeScenarios.java b/integration_tests/android_tests/src/main/java/org/apache/fory/android/AndroidForyRuntimeScenarios.java index ed1cb3ea01..54f84806f5 100644 --- a/integration_tests/android_tests/src/main/java/org/apache/fory/android/AndroidForyRuntimeScenarios.java +++ b/integration_tests/android_tests/src/main/java/org/apache/fory/android/AndroidForyRuntimeScenarios.java @@ -64,12 +64,8 @@ public static void androidRuntimeDisablesCodegenAndUnsafeCopies() { "async compilation must be disabled on Android"); MemoryBuffer buffer = MemoryUtils.buffer(16); - try { - buffer.copyToUnsafe(0, new byte[16], 0, 1); - fail("copyToUnsafe should fail on Android"); - } catch (UnsupportedOperationException expected) { - check(expected.getMessage().contains("Android"), expected.getMessage()); - } + byte[] target = new byte[16]; + buffer.copyToByteArray(0, target, 0, 1); } public static void structEnumCollectionAndMapRoundTrip() { diff --git a/java/fory-core/pom.xml b/java/fory-core/pom.xml index 63ee5f6c89..6e74e2c2a4 100644 --- a/java/fory-core/pom.xml +++ b/java/fory-core/pom.xml @@ -243,8 +243,6 @@ - - @@ -262,12 +260,6 @@ - - @@ -292,12 +284,6 @@ - - diff --git a/java/fory-core/src/main/java/org/apache/fory/builder/CodecBuilder.java b/java/fory-core/src/main/java/org/apache/fory/builder/CodecBuilder.java index 5c598b72ce..3bcad7a701 100644 --- a/java/fory-core/src/main/java/org/apache/fory/builder/CodecBuilder.java +++ b/java/fory-core/src/main/java/org/apache/fory/builder/CodecBuilder.java @@ -58,6 +58,7 @@ import org.apache.fory.platform.GraalvmSupport; import org.apache.fory.platform.JdkVersion; import org.apache.fory.platform.UnsafeOps; +import org.apache.fory.reflect.FieldAccessor; import org.apache.fory.reflect.ObjectCreator; import org.apache.fory.reflect.ObjectCreators; import org.apache.fory.reflect.ReflectionUtils; @@ -310,6 +311,20 @@ private Expression reflectAccessField( private Expression unsafeAccessField( Expression inputObject, Class cls, Descriptor descriptor) { String fieldName = descriptor.getName(); + if (JdkVersion.MAJOR_VERSION >= 25) { + Reference fieldAccessor = getFieldAccessor(descriptor); + boolean fieldNullable = fieldNullable(descriptor); + if (descriptor.getTypeRef().isPrimitive()) { + Preconditions.checkArgument(!fieldNullable); + TypeRef returnType = descriptor.getTypeRef(); + String funcName = "get" + StringUtils.capitalize(descriptor.getRawType().toString()); + return new Invoke(fieldAccessor, funcName, returnType, false, inputObject); + } else { + Invoke getObj = + new Invoke(fieldAccessor, "getObject", OBJECT_TYPE, fieldNullable, inputObject); + return tryCastIfPublic(getObj, descriptor.getTypeRef(), fieldName); + } + } Expression fieldOffsetExpr = fieldOffsetExpr(cls, descriptor); boolean fieldNullable = fieldNullable(descriptor); if (descriptor.getTypeRef().isPrimitive()) { @@ -345,7 +360,7 @@ private Expression fieldOffsetExpr(Class cls, Descriptor descriptor) { () -> { Expression classExpr = beanClassExpr(field.getDeclaringClass()); new Invoke(classExpr, "getDeclaredField", TypeRef.of(Field.class)); - Expression reflectFieldRef = getReflectField(cls, field, false); + Expression reflectFieldRef = getReflectField(field.getDeclaringClass(), field, false); return new StaticInvoke( UnsafeOps.class, "objectFieldOffset", PRIMITIVE_LONG_TYPE, reflectFieldRef) .inline(); @@ -355,6 +370,27 @@ private Expression fieldOffsetExpr(Class cls, Descriptor descriptor) { } } + private Reference getFieldAccessor(Descriptor descriptor) { + Field field = descriptor.getField(); + String fieldName = descriptor.getName(); + String fieldAccessorName = + (duplicatedFields.contains(fieldName) + ? field.getDeclaringClass().getName().replaceAll("\\.|\\$", "_") + "_" + : "") + + fieldName + + "_accessor_"; + return getOrCreateField( + true, + FieldAccessor.class, + fieldAccessorName, + () -> + new StaticInvoke( + FieldAccessor.class, + "createAccessor", + TypeRef.of(FieldAccessor.class), + getReflectField(field.getDeclaringClass(), field, false))); + } + /** * Returns an expression that deserialize data as a java bean of type {@link * CodecBuilder#beanClass}. @@ -423,6 +459,16 @@ private Expression reflectSetField(Expression bean, Field field, Expression valu */ private Expression unsafeSetField(Expression bean, Descriptor descriptor, Expression value) { TypeRef fieldType = descriptor.getTypeRef(); + if (JdkVersion.MAJOR_VERSION >= 25) { + Reference fieldAccessor = getFieldAccessor(descriptor); + if (descriptor.getTypeRef().isPrimitive()) { + Preconditions.checkArgument(getRawType(value.type()) == getRawType(fieldType)); + String funcName = "put" + StringUtils.capitalize(getRawType(fieldType).toString()); + return new Invoke(fieldAccessor, funcName, bean, value); + } else { + return new Invoke(fieldAccessor, "putObject", bean, value); + } + } // Use Field in case the class has duplicate field name as `fieldName`. Expression fieldOffsetExpr = fieldOffsetExpr(beanClass, descriptor); if (descriptor.getTypeRef().isPrimitive()) { @@ -442,7 +488,11 @@ private Reference getReflectField(Class cls, Field field, boolean setAccessib String fieldName = field.getName(); String fieldRefName; if (duplicatedFields.contains(fieldName)) { - fieldRefName = cls.getName().replaceAll("\\.|\\$", "_") + "_" + fieldName + "_Field"; + fieldRefName = + field.getDeclaringClass().getName().replaceAll("\\.|\\$", "_") + + "_" + + fieldName + + "_Field"; } else { fieldRefName = fieldName + "_Field"; } @@ -452,7 +502,11 @@ private Reference getReflectField(Class cls, Field field, boolean setAccessib fieldRefName, () -> { TypeRef fieldTypeRef = TypeRef.of(Field.class); - Expression classExpr = beanClassExpr(field.getDeclaringClass()); + Class declaringClass = field.getDeclaringClass(); + Expression classExpr = + staticClassFieldExpr( + declaringClass, + declaringClass.getName().replaceAll("\\.|\\$", "_") + "__class__"); Expression fieldExpr; if (GraalvmSupport.isGraalBuildTime()) { fieldExpr = @@ -486,12 +540,15 @@ protected Reference getOrCreateField( /** Returns an Expression that create a new java object of type {@link CodecBuilder#beanClass}. */ protected Expression newBean() { // TODO allow default access-level class. - if (sourcePublicAccessible(beanClass)) { + if (sourcePublicAccessible(beanClass) + && (JdkVersion.MAJOR_VERSION < 25 + || ReflectionUtils.hasPublicNoArgConstructor(beanClass))) { return new Expression.NewInstance(beanType); } else { - if (GraalvmSupport.IN_GRAALVM_NATIVE_IMAGE && JdkVersion.MAJOR_VERSION >= 25) { + if (JdkVersion.MAJOR_VERSION >= 25) { ObjectCreators.getObjectCreator(beanClass); // trigger cache - return new Invoke(getObjectCreator(beanClass), "newInstance", OBJECT_TYPE); + Invoke newInstance = new Invoke(getObjectCreator(beanClass), "newInstance", OBJECT_TYPE); + return sourcePublicAccessible(beanClass) ? new Cast(newInstance, beanType) : newInstance; } return new StaticInvoke(UnsafeOps.class, "newInstance", OBJECT_TYPE, beanClassExpr()); } diff --git a/java/fory-core/src/main/java/org/apache/fory/builder/ObjectCodecBuilder.java b/java/fory-core/src/main/java/org/apache/fory/builder/ObjectCodecBuilder.java index 2220a53013..f23801f948 100644 --- a/java/fory-core/src/main/java/org/apache/fory/builder/ObjectCodecBuilder.java +++ b/java/fory-core/src/main/java/org/apache/fory/builder/ObjectCodecBuilder.java @@ -26,8 +26,12 @@ import static org.apache.fory.collection.Collections.ofHashSet; import static org.apache.fory.type.TypeUtils.OBJECT_ARRAY_TYPE; import static org.apache.fory.type.TypeUtils.OBJECT_TYPE; +import static org.apache.fory.type.TypeUtils.PRIMITIVE_BOOLEAN_TYPE; import static org.apache.fory.type.TypeUtils.PRIMITIVE_BYTE_ARRAY_TYPE; import static org.apache.fory.type.TypeUtils.PRIMITIVE_BYTE_TYPE; +import static org.apache.fory.type.TypeUtils.PRIMITIVE_CHAR_TYPE; +import static org.apache.fory.type.TypeUtils.PRIMITIVE_DOUBLE_TYPE; +import static org.apache.fory.type.TypeUtils.PRIMITIVE_FLOAT_TYPE; import static org.apache.fory.type.TypeUtils.PRIMITIVE_INT_TYPE; import static org.apache.fory.type.TypeUtils.PRIMITIVE_LONG_TYPE; import static org.apache.fory.type.TypeUtils.PRIMITIVE_SHORT_TYPE; @@ -60,6 +64,7 @@ import org.apache.fory.logging.Logger; import org.apache.fory.logging.LoggerFactory; import org.apache.fory.meta.TypeDef; +import org.apache.fory.platform.JdkVersion; import org.apache.fory.platform.UnsafeOps; import org.apache.fory.reflect.ObjectCreator; import org.apache.fory.reflect.ObjectCreators; @@ -376,6 +381,9 @@ protected int getNumPrimitiveFields(List> primitiveGroups) { private List serializePrimitivesUnCompressed( Expression bean, Expression buffer, List> primitiveGroups, int totalSize) { + if (JdkVersion.MAJOR_VERSION >= 25) { + return serializeRawPrimitivesIndexed(bean, buffer, primitiveGroups, totalSize); + } List expressions = new ArrayList<>(); int numPrimitiveFields = getNumPrimitiveFields(primitiveGroups); Literal totalSizeLiteral = new Literal(totalSize, PRIMITIVE_INT_TYPE); @@ -469,6 +477,9 @@ private List serializePrimitivesUnCompressed( private List serializePrimitivesCompressed( Expression bean, Expression buffer, List> primitiveGroups, int totalSize) { + if (JdkVersion.MAJOR_VERSION >= 25) { + return serializeCompressedIndexed(bean, buffer, primitiveGroups, totalSize); + } List expressions = new ArrayList<>(); // int/long may need extra one-byte for writing. int extraSize = 0; @@ -617,6 +628,324 @@ private List serializePrimitivesCompressed( return expressions; } + private List serializeRawPrimitivesIndexed( + Expression bean, Expression buffer, List> primitiveGroups, int totalSize) { + List expressions = new ArrayList<>(); + int numPrimitiveFields = getNumPrimitiveFields(primitiveGroups); + Literal totalSizeLiteral = new Literal(totalSize, PRIMITIVE_INT_TYPE); + expressions.add(new Invoke(buffer, "grow", totalSizeLiteral)); + Expression writerIndex = new Invoke(buffer, "writerIndex", "writerIndex", PRIMITIVE_INT_TYPE); + expressions.add(writerIndex); + int acc = 0; + for (List group : primitiveGroups) { + ListExpression groupExpressions = new ListExpression(); + for (Descriptor descriptor : group) { + int dispatchId = getNumericDescriptorDispatchId(descriptor); + Expression fieldValue = getFieldValue(bean, descriptor); + if (fieldValue instanceof Inlineable) { + ((Inlineable) fieldValue).inline(); + } + if (dispatchId == DispatchId.BOOL) { + groupExpressions.add( + bufferPutBoolean(buffer, getBufferIndex(writerIndex, acc), fieldValue)); + acc += 1; + } else if (dispatchId == DispatchId.INT8) { + groupExpressions.add(bufferPutByte(buffer, getBufferIndex(writerIndex, acc), fieldValue)); + acc += 1; + } else if (dispatchId == DispatchId.UINT8) { + groupExpressions.add( + bufferPutByte( + buffer, + getBufferIndex(writerIndex, acc), + primitiveByteValue(fieldValue, descriptor))); + acc += 1; + } else if (dispatchId == DispatchId.CHAR) { + groupExpressions.add(bufferPutChar(buffer, getBufferIndex(writerIndex, acc), fieldValue)); + acc += 2; + } else if (dispatchId == DispatchId.INT16) { + groupExpressions.add( + bufferPutInt16(buffer, getBufferIndex(writerIndex, acc), fieldValue)); + acc += 2; + } else if (dispatchId == DispatchId.UINT16) { + groupExpressions.add( + bufferPutInt16( + buffer, + getBufferIndex(writerIndex, acc), + primitiveShortValue(fieldValue, descriptor))); + acc += 2; + } else if (dispatchId == DispatchId.FLOAT16 || dispatchId == DispatchId.BFLOAT16) { + groupExpressions.add( + bufferPutInt16( + buffer, + getBufferIndex(writerIndex, acc), + new Invoke(fieldValue, "toBits", SHORT_TYPE))); + acc += 2; + } else if (dispatchId == DispatchId.INT32) { + groupExpressions.add( + bufferPutInt32(buffer, getBufferIndex(writerIndex, acc), fieldValue)); + acc += 4; + } else if (dispatchId == DispatchId.UINT32) { + groupExpressions.add( + bufferPutInt32( + buffer, + getBufferIndex(writerIndex, acc), + primitiveIntValue(fieldValue, descriptor))); + acc += 4; + } else if (dispatchId == DispatchId.INT64 || dispatchId == DispatchId.UINT64) { + groupExpressions.add( + bufferPutInt64(buffer, getBufferIndex(writerIndex, acc), fieldValue)); + acc += 8; + } else if (dispatchId == DispatchId.FLOAT32) { + groupExpressions.add( + bufferPutFloat32(buffer, getBufferIndex(writerIndex, acc), fieldValue)); + acc += 4; + } else if (dispatchId == DispatchId.FLOAT64) { + groupExpressions.add( + bufferPutFloat64(buffer, getBufferIndex(writerIndex, acc), fieldValue)); + acc += 8; + } else { + throw new IllegalStateException("Unsupported dispatchId: " + dispatchId); + } + } + if (hasFewFields() || numPrimitiveFields < 4) { + expressions.add(groupExpressions); + } else { + expressions.add( + objectCodecOptimizer.invokeGenerated( + ofHashSet(bean, buffer, writerIndex), groupExpressions, "writeFields")); + } + } + Expression increaseWriterIndex = + new Invoke( + buffer, + "_increaseWriterIndexUnsafe", + new Literal(totalSizeLiteral, PRIMITIVE_INT_TYPE)); + expressions.add(increaseWriterIndex); + return expressions; + } + + private List serializeCompressedIndexed( + Expression bean, Expression buffer, List> primitiveGroups, int totalSize) { + List expressions = new ArrayList<>(); + int extraSize = 0; + for (List group : primitiveGroups) { + for (Descriptor d : group) { + int id = getNumericDescriptorDispatchId(d); + if (id == DispatchId.INT32 + || id == DispatchId.VARINT32 + || id == DispatchId.VAR_UINT32 + || id == DispatchId.UINT32) { + extraSize += 4; + } else if (id == DispatchId.INT64 + || id == DispatchId.VARINT64 + || id == DispatchId.TAGGED_INT64 + || id == DispatchId.VAR_UINT64 + || id == DispatchId.TAGGED_UINT64 + || id == DispatchId.UINT64) { + extraSize += 1; + } + } + } + int growSize = totalSize + extraSize; + expressions.add(new Invoke(buffer, "grow", Literal.ofInt(growSize))); + int numPrimitiveFields = getNumPrimitiveFields(primitiveGroups); + for (List group : primitiveGroups) { + ListExpression groupExpressions = new ListExpression(); + Expression writerIndex = new Invoke(buffer, "writerIndex", "writerIndex", PRIMITIVE_INT_TYPE); + int acc = 0; + boolean compressStarted = false; + for (Descriptor descriptor : group) { + int dispatchId = getNumericDescriptorDispatchId(descriptor); + Expression fieldValue = getFieldValue(bean, descriptor); + if (fieldValue instanceof Inlineable) { + ((Inlineable) fieldValue).inline(); + } + if (dispatchId == DispatchId.BOOL) { + groupExpressions.add( + bufferPutBoolean(buffer, getBufferIndex(writerIndex, acc), fieldValue)); + acc += 1; + } else if (dispatchId == DispatchId.INT8) { + groupExpressions.add(bufferPutByte(buffer, getBufferIndex(writerIndex, acc), fieldValue)); + acc += 1; + } else if (dispatchId == DispatchId.UINT8) { + groupExpressions.add( + bufferPutByte( + buffer, + getBufferIndex(writerIndex, acc), + primitiveByteValue(fieldValue, descriptor))); + acc += 1; + } else if (dispatchId == DispatchId.CHAR) { + groupExpressions.add(bufferPutChar(buffer, getBufferIndex(writerIndex, acc), fieldValue)); + acc += 2; + } else if (dispatchId == DispatchId.INT16) { + groupExpressions.add( + bufferPutInt16(buffer, getBufferIndex(writerIndex, acc), fieldValue)); + acc += 2; + } else if (dispatchId == DispatchId.UINT16) { + groupExpressions.add( + bufferPutInt16( + buffer, + getBufferIndex(writerIndex, acc), + primitiveShortValue(fieldValue, descriptor))); + acc += 2; + } else if (dispatchId == DispatchId.FLOAT16 || dispatchId == DispatchId.BFLOAT16) { + groupExpressions.add( + bufferPutInt16( + buffer, + getBufferIndex(writerIndex, acc), + new Invoke(fieldValue, "toBits", SHORT_TYPE))); + acc += 2; + } else if (dispatchId == DispatchId.FLOAT32) { + groupExpressions.add( + bufferPutFloat32(buffer, getBufferIndex(writerIndex, acc), fieldValue)); + acc += 4; + } else if (dispatchId == DispatchId.FLOAT64) { + groupExpressions.add( + bufferPutFloat64(buffer, getBufferIndex(writerIndex, acc), fieldValue)); + acc += 8; + } else if (dispatchId == DispatchId.INT32) { + groupExpressions.add( + bufferPutInt32(buffer, getBufferIndex(writerIndex, acc), fieldValue)); + acc += 4; + } else if (dispatchId == DispatchId.UINT32) { + groupExpressions.add( + bufferPutInt32( + buffer, + getBufferIndex(writerIndex, acc), + primitiveIntValue(fieldValue, descriptor))); + acc += 4; + } else if (dispatchId == DispatchId.INT64 || dispatchId == DispatchId.UINT64) { + groupExpressions.add( + bufferPutInt64(buffer, getBufferIndex(writerIndex, acc), fieldValue)); + acc += 8; + } else if (dispatchId == DispatchId.VARINT32) { + if (!compressStarted) { + addIncWriterIndexExpr(groupExpressions, buffer, acc); + compressStarted = true; + } + groupExpressions.add(new Invoke(buffer, "_unsafeWriteVarInt32", fieldValue)); + } else if (dispatchId == DispatchId.VAR_UINT32) { + if (!compressStarted) { + addIncWriterIndexExpr(groupExpressions, buffer, acc); + compressStarted = true; + } + groupExpressions.add( + new Invoke( + buffer, "_unsafeWriteVarUInt32", primitiveIntValue(fieldValue, descriptor))); + } else if (dispatchId == DispatchId.VARINT64) { + if (!compressStarted) { + addIncWriterIndexExpr(groupExpressions, buffer, acc); + compressStarted = true; + } + groupExpressions.add(new Invoke(buffer, "writeVarInt64", fieldValue)); + } else if (dispatchId == DispatchId.TAGGED_INT64) { + if (!compressStarted) { + addIncWriterIndexExpr(groupExpressions, buffer, acc); + compressStarted = true; + } + groupExpressions.add(new Invoke(buffer, "writeTaggedInt64", fieldValue)); + } else if (dispatchId == DispatchId.VAR_UINT64) { + if (!compressStarted) { + addIncWriterIndexExpr(groupExpressions, buffer, acc); + compressStarted = true; + } + groupExpressions.add(new Invoke(buffer, "writeVarUInt64", fieldValue)); + } else if (dispatchId == DispatchId.TAGGED_UINT64) { + if (!compressStarted) { + addIncWriterIndexExpr(groupExpressions, buffer, acc); + compressStarted = true; + } + groupExpressions.add(new Invoke(buffer, "writeTaggedUInt64", fieldValue)); + } else { + throw new IllegalStateException("Unsupported dispatchId: " + dispatchId); + } + } + if (!compressStarted) { + addIncWriterIndexExpr(groupExpressions, buffer, acc); + } + if (hasFewFields() || numPrimitiveFields < 4) { + expressions.add(groupExpressions); + } else { + expressions.add( + objectCodecOptimizer.invokeGenerated( + ofHashSet(bean, buffer, writerIndex), groupExpressions, "writeFields")); + } + } + return expressions; + } + + private Expression bufferPutByte(Expression buffer, Expression index, Expression value) { + return new Invoke(buffer, "_unsafePutByte", index, value); + } + + private Expression bufferPutBoolean(Expression buffer, Expression index, Expression value) { + return new Invoke(buffer, "_unsafePutBoolean", index, value); + } + + private Expression bufferPutChar(Expression buffer, Expression index, Expression value) { + return new Invoke(buffer, "_unsafePutChar", index, value); + } + + private Expression bufferPutInt16(Expression buffer, Expression index, Expression value) { + return new Invoke(buffer, "_unsafePutInt16", index, value); + } + + private Expression bufferPutInt32(Expression buffer, Expression index, Expression value) { + return new Invoke(buffer, "_unsafePutInt32", index, value); + } + + private Expression bufferPutInt64(Expression buffer, Expression index, Expression value) { + return new Invoke(buffer, "_unsafePutInt64", index, value); + } + + private Expression bufferPutFloat32(Expression buffer, Expression index, Expression value) { + return bufferPutInt32( + buffer, + index, + new StaticInvoke(Float.class, "floatToRawIntBits", PRIMITIVE_INT_TYPE, value)); + } + + private Expression bufferPutFloat64(Expression buffer, Expression index, Expression value) { + return bufferPutInt64( + buffer, + index, + new StaticInvoke(Double.class, "doubleToRawLongBits", PRIMITIVE_LONG_TYPE, value)); + } + + private Expression bufferGetByte(Expression buffer, Expression index) { + return new Invoke(buffer, "_unsafeGetByte", PRIMITIVE_BYTE_TYPE, index); + } + + private Expression bufferGetBoolean(Expression buffer, Expression index) { + return new Invoke(buffer, "_unsafeGetBoolean", PRIMITIVE_BOOLEAN_TYPE, index); + } + + private Expression bufferGetChar(Expression buffer, Expression index) { + return new Invoke(buffer, "_unsafeGetChar", PRIMITIVE_CHAR_TYPE, index); + } + + private Expression bufferGetInt16(Expression buffer, Expression index) { + return new Invoke(buffer, "_unsafeGetInt16", PRIMITIVE_SHORT_TYPE, index); + } + + private Expression bufferGetInt32(Expression buffer, Expression index) { + return new Invoke(buffer, "_unsafeGetInt32", PRIMITIVE_INT_TYPE, index); + } + + private Expression bufferGetInt64(Expression buffer, Expression index) { + return new Invoke(buffer, "_unsafeGetInt64", PRIMITIVE_LONG_TYPE, index); + } + + private Expression bufferGetFloat32(Expression buffer, Expression index) { + return new StaticInvoke( + Float.class, "intBitsToFloat", PRIMITIVE_FLOAT_TYPE, bufferGetInt32(buffer, index)); + } + + private Expression bufferGetFloat64(Expression buffer, Expression index) { + return new StaticInvoke( + Double.class, "longBitsToDouble", PRIMITIVE_DOUBLE_TYPE, bufferGetInt64(buffer, index)); + } + private Expression primitiveByteValue(Expression fieldValue, Descriptor descriptor) { return fieldValue.type().isPrimitive() ? cast(fieldValue, PRIMITIVE_BYTE_TYPE) @@ -869,6 +1198,7 @@ protected Expression createRecord(SortedMap recordComponent protected Expression createConstructorObject(FieldsArray fieldValues) { Expression[] params = new Expression[constructorFieldIndexes.length]; + Expression[] directParams = new Expression[constructorFieldIndexes.length]; for (int i = 0; i < constructorFieldIndexes.length; i++) { int index = constructorFieldIndexes[i]; if (index < 0) { @@ -876,9 +1206,16 @@ protected Expression createConstructorObject(FieldsArray fieldValues) { } else { params[i] = fieldValue(fieldValues, index); } + directParams[i] = tryInlineCast(params[i], TypeRef.of(constructorFieldTypes[i])); + } + ObjectCreator objectCreator = ObjectCreators.getObjectCreator(beanClass); + if (JdkVersion.MAJOR_VERSION >= 25 + && objectCreator.isOnlyPublicConstructor() + && sourcePublicAccessible(beanClass) + && constructorParamsAccessible()) { + return new NewInstance(beanType, directParams); } Expression args = new Expression.NewArray(OBJECT_ARRAY_TYPE, params); - ObjectCreators.getObjectCreator(beanClass); // trigger cache and make error raised early Expression newInstance = new Invoke(getObjectCreator(beanClass), "newInstanceWithArguments", OBJECT_TYPE, args); return sourcePublicAccessible(beanClass) ? new Cast(newInstance, beanType) : newInstance; @@ -894,6 +1231,15 @@ protected Expression defaultConstructorValue(int constructorParameterIndex) { "constructorFieldClass" + constructorParameterIndex + "_")); } + private boolean constructorParamsAccessible() { + for (Class constructorFieldType : constructorFieldTypes) { + if (!sourcePublicAccessible(constructorFieldType)) { + return false; + } + } + return true; + } + private void addNonConstructorFieldSetters( ListExpression expressions, Expression bean, FieldsArray fieldValues) { for (Descriptor descriptor : objectCodecOptimizer.descriptorGrouper.getSortedDescriptors()) { @@ -1109,6 +1455,9 @@ protected List deserializePrimitives( private List deserializeUnCompressedPrimitives( Expression bean, Expression buffer, List> primitiveGroups, int totalSize) { + if (JdkVersion.MAJOR_VERSION >= 25) { + return deserializeRawIndexed(bean, buffer, primitiveGroups, totalSize); + } List expressions = new ArrayList<>(); int numPrimitiveFields = getNumPrimitiveFields(primitiveGroups); Literal totalSizeLiteral = Literal.ofInt(totalSize); @@ -1213,6 +1562,9 @@ private List deserializeUnCompressedPrimitives( private List deserializeCompressedPrimitives( Expression bean, Expression buffer, List> primitiveGroups) { + if (JdkVersion.MAJOR_VERSION >= 25) { + return deserializeCompressedIndexed(bean, buffer, primitiveGroups); + } List expressions = new ArrayList<>(); int numPrimitiveFields = getNumPrimitiveFields(primitiveGroups); for (List group : primitiveGroups) { @@ -1360,6 +1712,247 @@ private List deserializeCompressedPrimitives( return expressions; } + private List deserializeRawIndexed( + Expression bean, Expression buffer, List> primitiveGroups, int totalSize) { + List expressions = new ArrayList<>(); + int numPrimitiveFields = getNumPrimitiveFields(primitiveGroups); + Literal totalSizeLiteral = Literal.ofInt(totalSize); + expressions.add(new Invoke(buffer, "checkReadableBytes", totalSizeLiteral)); + Expression readerIndex = new Invoke(buffer, "readerIndex", "readerIndex", PRIMITIVE_INT_TYPE); + expressions.add(readerIndex); + int acc = 0; + for (List group : primitiveGroups) { + ListExpression groupExpressions = new ListExpression(); + for (Descriptor descriptor : group) { + int dispatchId = getNumericDescriptorDispatchId(descriptor); + Expression fieldValue; + if (dispatchId == DispatchId.BOOL) { + fieldValue = bufferGetBoolean(buffer, getBufferIndex(readerIndex, acc)); + acc += 1; + } else if (dispatchId == DispatchId.INT8) { + fieldValue = bufferGetByte(buffer, getBufferIndex(readerIndex, acc)); + acc += 1; + } else if (dispatchId == DispatchId.UINT8) { + fieldValue = + new StaticInvoke( + Byte.class, + "toUnsignedInt", + descriptor.getTypeRef(), + bufferGetByte(buffer, getBufferIndex(readerIndex, acc))); + acc += 1; + } else if (dispatchId == DispatchId.CHAR) { + fieldValue = bufferGetChar(buffer, getBufferIndex(readerIndex, acc)); + acc += 2; + } else if (dispatchId == DispatchId.INT16) { + fieldValue = bufferGetInt16(buffer, getBufferIndex(readerIndex, acc)); + acc += 2; + } else if (dispatchId == DispatchId.UINT16) { + fieldValue = + new StaticInvoke( + Short.class, + "toUnsignedInt", + descriptor.getTypeRef(), + bufferGetInt16(buffer, getBufferIndex(readerIndex, acc))); + acc += 2; + } else if (dispatchId == DispatchId.FLOAT16) { + fieldValue = + new StaticInvoke( + Float16.class, + "fromBits", + TypeRef.of(Float16.class), + bufferGetInt16(buffer, getBufferIndex(readerIndex, acc))); + acc += 2; + } else if (dispatchId == DispatchId.BFLOAT16) { + fieldValue = + new StaticInvoke( + BFloat16.class, + "fromBits", + TypeRef.of(BFloat16.class), + bufferGetInt16(buffer, getBufferIndex(readerIndex, acc))); + acc += 2; + } else if (dispatchId == DispatchId.INT32) { + fieldValue = bufferGetInt32(buffer, getBufferIndex(readerIndex, acc)); + acc += 4; + } else if (dispatchId == DispatchId.UINT32) { + fieldValue = + new StaticInvoke( + Integer.class, + "toUnsignedLong", + descriptor.getTypeRef(), + bufferGetInt32(buffer, getBufferIndex(readerIndex, acc))); + acc += 4; + } else if (dispatchId == DispatchId.INT64 || dispatchId == DispatchId.UINT64) { + fieldValue = bufferGetInt64(buffer, getBufferIndex(readerIndex, acc)); + acc += 8; + } else if (dispatchId == DispatchId.FLOAT32) { + fieldValue = bufferGetFloat32(buffer, getBufferIndex(readerIndex, acc)); + acc += 4; + } else if (dispatchId == DispatchId.FLOAT64) { + fieldValue = bufferGetFloat64(buffer, getBufferIndex(readerIndex, acc)); + acc += 8; + } else { + throw new IllegalStateException("Unsupported dispatchId: " + dispatchId); + } + groupExpressions.add(setFieldValue(bean, descriptor, fieldValue)); + } + if (hasFewFields() || numPrimitiveFields < 4 || isRecord) { + expressions.add(groupExpressions); + } else { + expressions.add( + objectCodecOptimizer.invokeGenerated( + ofHashSet(bean, buffer, readerIndex), groupExpressions, "readFields")); + } + } + Expression increaseReaderIndex = + new Invoke( + buffer, "increaseReaderIndex", new Literal(totalSizeLiteral, PRIMITIVE_INT_TYPE)); + expressions.add(increaseReaderIndex); + return expressions; + } + + private List deserializeCompressedIndexed( + Expression bean, Expression buffer, List> primitiveGroups) { + List expressions = new ArrayList<>(); + int numPrimitiveFields = getNumPrimitiveFields(primitiveGroups); + for (List group : primitiveGroups) { + ReplaceStub checkReadableBytesStub = new ReplaceStub(); + expressions.add(checkReadableBytesStub); + Expression readerIndex = new Invoke(buffer, "readerIndex", "readerIndex", PRIMITIVE_INT_TYPE); + expressions.add(readerIndex); + ListExpression groupExpressions = new ListExpression(); + int acc = 0; + boolean compressStarted = false; + for (Descriptor descriptor : group) { + int dispatchId = getNumericDescriptorDispatchId(descriptor); + Expression fieldValue; + if (dispatchId == DispatchId.BOOL) { + fieldValue = bufferGetBoolean(buffer, getBufferIndex(readerIndex, acc)); + acc += 1; + } else if (dispatchId == DispatchId.INT8) { + fieldValue = bufferGetByte(buffer, getBufferIndex(readerIndex, acc)); + acc += 1; + } else if (dispatchId == DispatchId.UINT8) { + fieldValue = + new StaticInvoke( + Byte.class, + "toUnsignedInt", + descriptor.getTypeRef(), + bufferGetByte(buffer, getBufferIndex(readerIndex, acc))); + acc += 1; + } else if (dispatchId == DispatchId.CHAR) { + fieldValue = bufferGetChar(buffer, getBufferIndex(readerIndex, acc)); + acc += 2; + } else if (dispatchId == DispatchId.INT16) { + fieldValue = bufferGetInt16(buffer, getBufferIndex(readerIndex, acc)); + acc += 2; + } else if (dispatchId == DispatchId.UINT16) { + fieldValue = + new StaticInvoke( + Short.class, + "toUnsignedInt", + descriptor.getTypeRef(), + bufferGetInt16(buffer, getBufferIndex(readerIndex, acc))); + acc += 2; + } else if (dispatchId == DispatchId.FLOAT16) { + fieldValue = + new StaticInvoke( + Float16.class, + "fromBits", + TypeRef.of(Float16.class), + bufferGetInt16(buffer, getBufferIndex(readerIndex, acc))); + acc += 2; + } else if (dispatchId == DispatchId.BFLOAT16) { + fieldValue = + new StaticInvoke( + BFloat16.class, + "fromBits", + TypeRef.of(BFloat16.class), + bufferGetInt16(buffer, getBufferIndex(readerIndex, acc))); + acc += 2; + } else if (dispatchId == DispatchId.FLOAT32) { + fieldValue = bufferGetFloat32(buffer, getBufferIndex(readerIndex, acc)); + acc += 4; + } else if (dispatchId == DispatchId.FLOAT64) { + fieldValue = bufferGetFloat64(buffer, getBufferIndex(readerIndex, acc)); + acc += 8; + } else if (dispatchId == DispatchId.INT32) { + fieldValue = bufferGetInt32(buffer, getBufferIndex(readerIndex, acc)); + acc += 4; + } else if (dispatchId == DispatchId.UINT32) { + fieldValue = + new StaticInvoke( + Integer.class, + "toUnsignedLong", + descriptor.getTypeRef(), + bufferGetInt32(buffer, getBufferIndex(readerIndex, acc))); + acc += 4; + } else if (dispatchId == DispatchId.INT64 || dispatchId == DispatchId.UINT64) { + fieldValue = bufferGetInt64(buffer, getBufferIndex(readerIndex, acc)); + acc += 8; + } else if (dispatchId == DispatchId.VARINT32) { + if (!compressStarted) { + compressStarted = true; + addIncReaderIndexExpr(groupExpressions, buffer, acc); + } + fieldValue = readVarInt32(buffer); + } else if (dispatchId == DispatchId.VAR_UINT32) { + if (!compressStarted) { + compressStarted = true; + addIncReaderIndexExpr(groupExpressions, buffer, acc); + } + fieldValue = + new StaticInvoke( + Integer.class, + "toUnsignedLong", + descriptor.getTypeRef(), + new Invoke(buffer, "readVarUInt32", PRIMITIVE_INT_TYPE)); + } else if (dispatchId == DispatchId.VARINT64) { + if (!compressStarted) { + compressStarted = true; + addIncReaderIndexExpr(groupExpressions, buffer, acc); + } + fieldValue = new Invoke(buffer, "readVarInt64", PRIMITIVE_LONG_TYPE); + } else if (dispatchId == DispatchId.TAGGED_INT64) { + if (!compressStarted) { + compressStarted = true; + addIncReaderIndexExpr(groupExpressions, buffer, acc); + } + fieldValue = new Invoke(buffer, "readTaggedInt64", PRIMITIVE_LONG_TYPE); + } else if (dispatchId == DispatchId.VAR_UINT64) { + if (!compressStarted) { + compressStarted = true; + addIncReaderIndexExpr(groupExpressions, buffer, acc); + } + fieldValue = new Invoke(buffer, "readVarUInt64", PRIMITIVE_LONG_TYPE); + } else if (dispatchId == DispatchId.TAGGED_UINT64) { + if (!compressStarted) { + compressStarted = true; + addIncReaderIndexExpr(groupExpressions, buffer, acc); + } + fieldValue = new Invoke(buffer, "readTaggedUInt64", PRIMITIVE_LONG_TYPE); + } else { + throw new IllegalStateException("Unsupported dispatchId: " + dispatchId); + } + groupExpressions.add(setFieldValue(bean, descriptor, fieldValue)); + } + if (acc != 0) { + checkReadableBytesStub.setTargetObject( + new Invoke(buffer, "checkReadableBytes", Literal.ofInt(acc))); + } + if (!compressStarted) { + addIncReaderIndexExpr(groupExpressions, buffer, acc); + } + if (hasFewFields() || numPrimitiveFields < 4 || isRecord) { + expressions.add(groupExpressions); + } else { + expressions.add( + objectCodecOptimizer.invokeGenerated( + ofHashSet(bean, buffer, readerIndex), groupExpressions, "readFields")); + } + } + return expressions; + } + private void addIncReaderIndexExpr(ListExpression expressions, Expression buffer, int diff) { if (diff != 0) { expressions.add(new Invoke(buffer, "increaseReaderIndex", Literal.ofInt(diff))); @@ -1372,4 +1965,11 @@ private Expression getReaderAddress(Expression readerPos, long acc) { } return add(readerPos, new Literal(acc, PRIMITIVE_LONG_TYPE)); } + + private Expression getBufferIndex(Expression index, int acc) { + if (acc == 0) { + return index; + } + return add(index, Literal.ofInt(acc)); + } } diff --git a/java/fory-core/src/main/java/org/apache/fory/memory/MemoryBuffer.java b/java/fory-core/src/main/java/org/apache/fory/memory/MemoryBuffer.java index 47d69b6ee2..743ccd32bc 100644 --- a/java/fory-core/src/main/java/org/apache/fory/memory/MemoryBuffer.java +++ b/java/fory-core/src/main/java/org/apache/fory/memory/MemoryBuffer.java @@ -3625,18 +3625,134 @@ public void copyFrom(int offset, MemoryBuffer source, int sourcePointer, int num source.copyTo(sourcePointer, this, offset, numBytes); } - /** - * JVM-only bulk copy method. Copies {@code numBytes} bytes to target unsafe object and pointer. - * Throws on Android before executing unsafe memory access. - */ - public void copyToUnsafe(long offset, Object target, long targetPointer, int numBytes) { + public void copyToByteArray(int offset, byte[] target, int targetOffset, int numBytes) { if (AndroidSupport.IS_ANDROID) { - MemoryOps.throwRawUnsafeMemoryCopyUnsupported(); + MemoryOps.copyToByteArray(this, offset, target, targetOffset, numBytes); } else { - final long thisPointer = this.address + offset; - checkArgument(thisPointer + numBytes <= addressLimit); - UNSAFE.copyMemory(this.heapMemory, thisPointer, target, targetPointer, numBytes); + checkArrayCopy(offset, targetOffset, target.length, numBytes, 0); + UNSAFE.copyMemory( + heapMemory, + address + offset, + target, + UnsafeOps.BYTE_ARRAY_OFFSET + targetOffset, + numBytes); + } + } + + public void copyToBooleanArray(int offset, boolean[] target, int targetOffset, int numBytes) { + if (AndroidSupport.IS_ANDROID) { + MemoryOps.copyToBooleanArray(this, offset, target, targetOffset, numBytes); + } else { + checkArrayCopy(offset, targetOffset, target.length, numBytes, 0); + UNSAFE.copyMemory( + heapMemory, + address + offset, + target, + UnsafeOps.BOOLEAN_ARRAY_OFFSET + targetOffset, + numBytes); + } + } + + public void copyToCharArray(int offset, char[] target, int targetOffset, int numBytes) { + if (AndroidSupport.IS_ANDROID) { + MemoryOps.copyToCharArray(this, offset, target, targetOffset, numBytes); + } else { + checkArrayCopy(offset, targetOffset, target.length, numBytes, 1); + UNSAFE.copyMemory( + heapMemory, + address + offset, + target, + UnsafeOps.CHAR_ARRAY_OFFSET + arrayCopyOffset(targetOffset, 1), + numBytes); + } + } + + public void copyToShortArray(int offset, short[] target, int targetOffset, int numBytes) { + if (AndroidSupport.IS_ANDROID) { + MemoryOps.copyToShortArray(this, offset, target, targetOffset, numBytes); + } else { + checkArrayCopy(offset, targetOffset, target.length, numBytes, 1); + UNSAFE.copyMemory( + heapMemory, + address + offset, + target, + UnsafeOps.SHORT_ARRAY_OFFSET + arrayCopyOffset(targetOffset, 1), + numBytes); + } + } + + public void copyToIntArray(int offset, int[] target, int targetOffset, int numBytes) { + if (AndroidSupport.IS_ANDROID) { + MemoryOps.copyToIntArray(this, offset, target, targetOffset, numBytes); + } else { + checkArrayCopy(offset, targetOffset, target.length, numBytes, 2); + UNSAFE.copyMemory( + heapMemory, + address + offset, + target, + UnsafeOps.INT_ARRAY_OFFSET + arrayCopyOffset(targetOffset, 2), + numBytes); + } + } + + public void copyToLongArray(int offset, long[] target, int targetOffset, int numBytes) { + if (AndroidSupport.IS_ANDROID) { + MemoryOps.copyToLongArray(this, offset, target, targetOffset, numBytes); + } else { + checkArrayCopy(offset, targetOffset, target.length, numBytes, 3); + UNSAFE.copyMemory( + heapMemory, + address + offset, + target, + UnsafeOps.LONG_ARRAY_OFFSET + arrayCopyOffset(targetOffset, 3), + numBytes); + } + } + + public void copyToFloatArray(int offset, float[] target, int targetOffset, int numBytes) { + if (AndroidSupport.IS_ANDROID) { + MemoryOps.copyToFloatArray(this, offset, target, targetOffset, numBytes); + } else { + checkArrayCopy(offset, targetOffset, target.length, numBytes, 2); + UNSAFE.copyMemory( + heapMemory, + address + offset, + target, + UnsafeOps.FLOAT_ARRAY_OFFSET + arrayCopyOffset(targetOffset, 2), + numBytes); + } + } + + public void copyToDoubleArray(int offset, double[] target, int targetOffset, int numBytes) { + if (AndroidSupport.IS_ANDROID) { + MemoryOps.copyToDoubleArray(this, offset, target, targetOffset, numBytes); + } else { + checkArrayCopy(offset, targetOffset, target.length, numBytes, 3); + UNSAFE.copyMemory( + heapMemory, + address + offset, + target, + UnsafeOps.DOUBLE_ARRAY_OFFSET + arrayCopyOffset(targetOffset, 3), + numBytes); + } + } + + private void checkArrayCopy( + int offset, int targetOffset, int targetLength, int numBytes, int elementShift) { + int elementMask = (1 << elementShift) - 1; + if ((numBytes & elementMask) != 0) { + throw new IllegalArgumentException("numBytes is not aligned to array element size"); } + int numElements = numBytes >> elementShift; + if ((offset | targetOffset | numBytes | numElements) < 0 + || offset > size - numBytes + || targetOffset > targetLength - numElements) { + throwOOBException(); + } + } + + private static long arrayCopyOffset(int elementOffset, int elementShift) { + return (long) elementOffset << elementShift; } /** diff --git a/java/fory-core/src/main/java/org/apache/fory/memory/MemoryOps.java b/java/fory-core/src/main/java/org/apache/fory/memory/MemoryOps.java index 1b235c5b2f..8b1879f250 100644 --- a/java/fory-core/src/main/java/org/apache/fory/memory/MemoryOps.java +++ b/java/fory-core/src/main/java/org/apache/fory/memory/MemoryOps.java @@ -1259,6 +1259,108 @@ static void copyTo( offset, targetOffset, numBytes, source.size, target.size)); } + static void copyToByteArray( + MemoryBuffer source, int offset, byte[] target, int targetOffset, int numBytes) { + checkArrayCopy(source, offset, targetOffset, target.length, numBytes, 0); + copy(source.heapMemory, heapIndex(source, offset), target, targetOffset, numBytes); + } + + static void copyToBooleanArray( + MemoryBuffer source, int offset, boolean[] target, int targetOffset, int numBytes) { + checkArrayCopy(source, offset, targetOffset, target.length, numBytes, 0); + byte[] bytes = source.heapMemory; + int sourceIndex = heapIndex(source, offset); + for (int i = 0; i < numBytes; i++) { + target[targetOffset + i] = bytes[sourceIndex + i] != 0; + } + } + + static void copyToCharArray( + MemoryBuffer source, int offset, char[] target, int targetOffset, int numBytes) { + checkArrayCopy(source, offset, targetOffset, target.length, numBytes, 1); + byte[] bytes = source.heapMemory; + int sourceIndex = heapIndex(source, offset); + int numElements = numBytes >>> 1; + for (int i = 0; i < numElements; i++, sourceIndex += 2) { + target[targetOffset + i] = (char) getInt16(bytes, sourceIndex); + } + } + + static void copyToShortArray( + MemoryBuffer source, int offset, short[] target, int targetOffset, int numBytes) { + checkArrayCopy(source, offset, targetOffset, target.length, numBytes, 1); + byte[] bytes = source.heapMemory; + int sourceIndex = heapIndex(source, offset); + int numElements = numBytes >>> 1; + for (int i = 0; i < numElements; i++, sourceIndex += 2) { + target[targetOffset + i] = getInt16(bytes, sourceIndex); + } + } + + static void copyToIntArray( + MemoryBuffer source, int offset, int[] target, int targetOffset, int numBytes) { + checkArrayCopy(source, offset, targetOffset, target.length, numBytes, 2); + byte[] bytes = source.heapMemory; + int sourceIndex = heapIndex(source, offset); + int numElements = numBytes >>> 2; + for (int i = 0; i < numElements; i++, sourceIndex += 4) { + target[targetOffset + i] = getInt32(bytes, sourceIndex); + } + } + + static void copyToLongArray( + MemoryBuffer source, int offset, long[] target, int targetOffset, int numBytes) { + checkArrayCopy(source, offset, targetOffset, target.length, numBytes, 3); + byte[] bytes = source.heapMemory; + int sourceIndex = heapIndex(source, offset); + int numElements = numBytes >>> 3; + for (int i = 0; i < numElements; i++, sourceIndex += 8) { + target[targetOffset + i] = getInt64(bytes, sourceIndex); + } + } + + static void copyToFloatArray( + MemoryBuffer source, int offset, float[] target, int targetOffset, int numBytes) { + checkArrayCopy(source, offset, targetOffset, target.length, numBytes, 2); + byte[] bytes = source.heapMemory; + int sourceIndex = heapIndex(source, offset); + int numElements = numBytes >>> 2; + for (int i = 0; i < numElements; i++, sourceIndex += 4) { + target[targetOffset + i] = getFloat32(bytes, sourceIndex); + } + } + + static void copyToDoubleArray( + MemoryBuffer source, int offset, double[] target, int targetOffset, int numBytes) { + checkArrayCopy(source, offset, targetOffset, target.length, numBytes, 3); + byte[] bytes = source.heapMemory; + int sourceIndex = heapIndex(source, offset); + int numElements = numBytes >>> 3; + for (int i = 0; i < numElements; i++, sourceIndex += 8) { + target[targetOffset + i] = getFloat64(bytes, sourceIndex); + } + } + + private static void checkArrayCopy( + MemoryBuffer source, + int offset, + int targetOffset, + int targetLength, + int numBytes, + int shift) { + checkHeap(source); + int mask = (1 << shift) - 1; + if ((numBytes & mask) != 0) { + throw new IllegalArgumentException("numBytes is not aligned to array element size"); + } + int numElements = numBytes >>> shift; + if ((offset | targetOffset | numBytes | numElements) < 0 + || offset > source.size - numBytes + || targetOffset > targetLength - numElements) { + throwOOBException(source); + } + } + static boolean equalTo( MemoryBuffer buffer, MemoryBuffer other, int offset1, int offset2, int len) { checkArgument(offset1 >= 0 && offset1 <= buffer.size - len); diff --git a/java/fory-core/src/main/java25/org/apache/fory/builder/CodecBuilder.java b/java/fory-core/src/main/java25/org/apache/fory/builder/CodecBuilder.java deleted file mode 100644 index b74b523c0f..0000000000 --- a/java/fory-core/src/main/java25/org/apache/fory/builder/CodecBuilder.java +++ /dev/null @@ -1,732 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.fory.builder; - -import static org.apache.fory.codegen.Expression.Invoke.inlineInvoke; -import static org.apache.fory.type.TypeUtils.CLASS_TYPE; -import static org.apache.fory.type.TypeUtils.OBJECT_ARRAY_TYPE; -import static org.apache.fory.type.TypeUtils.OBJECT_TYPE; -import static org.apache.fory.type.TypeUtils.PRIMITIVE_BOOLEAN_TYPE; -import static org.apache.fory.type.TypeUtils.PRIMITIVE_BYTE_TYPE; -import static org.apache.fory.type.TypeUtils.PRIMITIVE_CHAR_TYPE; -import static org.apache.fory.type.TypeUtils.PRIMITIVE_DOUBLE_TYPE; -import static org.apache.fory.type.TypeUtils.PRIMITIVE_FLOAT_TYPE; -import static org.apache.fory.type.TypeUtils.PRIMITIVE_INT_TYPE; -import static org.apache.fory.type.TypeUtils.PRIMITIVE_LONG_TYPE; -import static org.apache.fory.type.TypeUtils.PRIMITIVE_SHORT_TYPE; -import static org.apache.fory.type.TypeUtils.PRIMITIVE_VOID_TYPE; -import static org.apache.fory.type.TypeUtils.getRawType; - -import java.lang.invoke.MethodHandle; -import java.lang.reflect.Field; -import java.lang.reflect.Modifier; -import java.util.HashMap; -import java.util.Map; -import java.util.Objects; -import java.util.Set; -import java.util.function.Supplier; -import java.util.stream.Collectors; -import org.apache.fory.codegen.CodegenContext; -import org.apache.fory.codegen.Expression; -import org.apache.fory.codegen.Expression.Cast; -import org.apache.fory.codegen.Expression.Inlineable; -import org.apache.fory.codegen.Expression.Invoke; -import org.apache.fory.codegen.Expression.ListExpression; -import org.apache.fory.codegen.Expression.Literal; -import org.apache.fory.codegen.Expression.Reference; -import org.apache.fory.codegen.Expression.StaticInvoke; -import org.apache.fory.collection.Tuple2; -import org.apache.fory.memory.MemoryBuffer; -import org.apache.fory.memory.NativeByteOrder; -import org.apache.fory.platform.GraalvmSupport; -import org.apache.fory.platform.UnsafeOps; -import org.apache.fory.reflect.FieldAccessor; -import org.apache.fory.reflect.ObjectCreator; -import org.apache.fory.reflect.ObjectCreators; -import org.apache.fory.reflect.ReflectionUtils; -import org.apache.fory.reflect.TypeRef; -import org.apache.fory.resolver.TypeInfo; -import org.apache.fory.resolver.TypeInfoHolder; -import org.apache.fory.type.Descriptor; -import org.apache.fory.util.Preconditions; -import org.apache.fory.util.StringUtils; -import org.apache.fory.util.function.Functions; -import org.apache.fory.util.record.RecordComponent; -import org.apache.fory.util.record.RecordUtils; - -/** - * Base builder for generating code to serialize java bean in row-format or object stream format. - * - *
    - * This builder has following requirements for the class of java bean: - *
  • public - *
  • For instance inner class, ignore outer class field. - *
  • For instance inner class, deserialized outer class field is null - *
- */ -@SuppressWarnings("UnstableApiUsage") -public abstract class CodecBuilder { - protected static final String ROOT_OBJECT_NAME = "_f_obj"; - static TypeRef objectArrayTypeRef = TypeRef.of(Object[].class); - static TypeRef bufferTypeRef = TypeRef.of(MemoryBuffer.class); - static TypeRef classInfoTypeRef = TypeRef.of(TypeInfo.class); - static TypeRef classInfoHolderTypeRef = TypeRef.of(TypeInfoHolder.class); - - protected final CodegenContext ctx; - protected final TypeRef beanType; - protected final Class beanClass; - protected final boolean isRecord; - protected final boolean isInterface; - private final Set duplicatedFields; - public static final Reference recordComponentDefaultValues = - new Reference("recordComponentDefaultValues", OBJECT_ARRAY_TYPE); - protected final Map fieldMap = new HashMap<>(); - protected boolean recordCtrAccessible; - - public CodecBuilder(CodegenContext ctx, TypeRef beanType) { - this.ctx = ctx; - this.beanType = beanType; - this.beanClass = getRawType(beanType); - isRecord = RecordUtils.isRecord(beanClass); - isInterface = beanClass.isInterface(); - if (isRecord) { - recordCtrAccessible = recordCtrAccessible(beanClass); - } - duplicatedFields = Descriptor.getSortedDuplicatedMembers(beanClass).keySet(); - // don't ctx.addImport beanClass, because it maybe causes name collide. - ctx.reserveName(ROOT_OBJECT_NAME); - // Don't import other packages to avoid class conflicts. - // For example user class named as `Date`/`List`/`MemoryBuffer` - // Skip Java reserved words since they can't be used as variable names anyway - // (e.g., Kotlin allows field names like "new" which are valid at bytecode level) - ReflectionUtils.getFields(beanType.getRawType(), true).stream() - .map(Field::getName) - .filter(name -> !CodegenContext.JAVA_RESERVED_WORDS.contains(name)) - .collect(Collectors.toSet()) - .forEach(ctx::reserveName); - } - - public abstract String codecClassName(Class cls); - - /** Generate codec class code. */ - public abstract String genCode(); - - /** Returns an expression that serialize java bean of type {@link CodecBuilder#beanClass}. */ - public abstract Expression buildEncodeExpression(); - - protected boolean sourcePublicAccessible(Class cls) { - return ctx.sourcePublicAccessible(cls); - } - - protected boolean fieldNullable(Descriptor descriptor) { - return false; - } - - protected Expression tryInlineCast(Expression expression, TypeRef targetType) { - return tryCastIfPublic(expression, targetType, true); - } - - protected Expression tryCastIfPublic(Expression expression, TypeRef targetType) { - return tryCastIfPublic(expression, targetType, false); - } - - protected Expression tryCastIfPublic( - Expression expression, TypeRef targetType, boolean inline) { - Class rawType = getRawType(targetType); - if (inline) { - if (sourcePublicAccessible(rawType)) { - return new Cast(expression, targetType); - } else { - return new Cast(expression, ReflectionUtils.getPublicSuperType(TypeRef.of(rawType))); - } - } - return tryCastIfPublic(expression, targetType, "castedValue"); - } - - protected Expression tryCastIfPublic( - Expression expression, TypeRef targetType, String valuePrefix) { - Class rawType = getRawType(targetType); - Class expressionRawType = getRawType(expression.type()); - // Source casts use erased Java types. Captured wildcard metadata can fail full generic subtype - // checks even when the emitted local variable already has an assignable raw type. - boolean rawTypeAlreadyAssignable = rawType.isAssignableFrom(expressionRawType); - if (sourcePublicAccessible(rawType) - && !expression.type().wrap().isSubtypeOf(targetType.wrap()) - && !rawTypeAlreadyAssignable) { - return new Cast(expression, targetType, valuePrefix); - } - if (rawType.isArray()) { - return new Cast(expression, OBJECT_ARRAY_TYPE, valuePrefix); - } - return expression; - } - - protected Reference getRecordCtrHandle() { - String fieldName = "_record_ctr_"; - Reference fieldRef = fieldMap.get(fieldName); - if (fieldRef == null) { - // trigger cache for graalvm - RecordUtils.getRecordCtrHandle(beanClass); - StaticInvoke getRecordCtrHandle = - new StaticInvoke( - RecordUtils.class, - "getRecordCtrHandle", - TypeRef.of(MethodHandle.class), - beanClassExpr()); - ctx.addField(ctx.type(MethodHandle.class), fieldName, getRecordCtrHandle); - fieldRef = new Reference(fieldName, TypeRef.of(MethodHandle.class)); - fieldMap.put(fieldName, fieldRef); - } - return fieldRef; - } - - protected Expression buildDefaultComponentsArray() { - return new StaticInvoke( - UnsafeOps.class, "copyObjectArray", OBJECT_ARRAY_TYPE, recordComponentDefaultValues); - } - - /** Returns an expression that get field value from bean. */ - protected Expression getFieldValue(Expression inputBeanExpr, Descriptor descriptor) { - TypeRef fieldType = descriptor.getTypeRef(); - Class rawType = descriptor.getRawType(); - String fieldName = descriptor.getName(); - boolean fieldNullable = fieldNullable(descriptor); - if (isInterface) { - return new Invoke(inputBeanExpr, descriptor.getName(), fieldName, fieldType, fieldNullable); - } - if (isRecord) { - return getRecordFieldValue(inputBeanExpr, descriptor); - } - if (duplicatedFields.contains(fieldName) || !Modifier.isPublic(beanClass.getModifiers())) { - return unsafeAccessField(inputBeanExpr, beanClass, descriptor); - } - if (!sourcePublicAccessible(rawType)) { - fieldType = OBJECT_TYPE; - } - // public field or non-private non-java field access field directly. - if (Modifier.isPublic(descriptor.getModifiers())) { - return new Expression.FieldValue(inputBeanExpr, fieldName, fieldType, fieldNullable, false); - } else if (descriptor.getReadMethod() != null - && Modifier.isPublic(descriptor.getReadMethod().getModifiers())) { - return new Invoke( - inputBeanExpr, descriptor.getReadMethod().getName(), fieldName, fieldType, fieldNullable); - } else { - if (!Modifier.isPrivate(descriptor.getModifiers())) { - if (AccessorHelper.defineAccessor(descriptor.getField())) { - return new StaticInvoke( - AccessorHelper.getAccessorClass(descriptor.getField()), - fieldName, - fieldType, - fieldNullable, - inputBeanExpr); - } - } - if (descriptor.getReadMethod() != null - && !Modifier.isPrivate(descriptor.getReadMethod().getModifiers())) { - if (AccessorHelper.defineAccessor(descriptor.getReadMethod())) { - return new StaticInvoke( - AccessorHelper.getAccessorClass(descriptor.getReadMethod()), - descriptor.getReadMethod().getName(), - fieldType, - fieldNullable, - inputBeanExpr); - } - } - return unsafeAccessField(inputBeanExpr, beanClass, descriptor); - } - } - - private Expression getRecordFieldValue(Expression inputBeanExpr, Descriptor descriptor) { - TypeRef fieldType = descriptor.getTypeRef(); - if (!sourcePublicAccessible(descriptor.getRawType())) { - fieldType = OBJECT_TYPE; - } - String fieldName = descriptor.getName(); - boolean fieldNullable = fieldNullable(descriptor); - if (Modifier.isPublic(beanClass.getModifiers())) { - Preconditions.checkNotNull(descriptor.getReadMethod()); - return new Invoke( - inputBeanExpr, descriptor.getReadMethod().getName(), fieldName, fieldType, fieldNullable); - } else { - String key = "_" + fieldName + "_getter_"; - Reference ref = fieldMap.get(key); - Tuple2, String> methodInfo = Functions.getterMethodInfo(descriptor.getRawType()); - if (ref == null) { - Class funcInterface = methodInfo.f0; - TypeRef getterType = TypeRef.of(funcInterface); - if (GraalvmSupport.isGraalBuildTime()) { - // generate getter ahead at native image build time. - Functions.makeGetterFunction(beanClass, fieldName); - } - Expression getter = - new StaticInvoke( - Functions.class, - "makeGetterFunction", - OBJECT_TYPE, - beanClassExpr(), - Literal.ofString(fieldName)); - getter = new Cast(getter, getterType); - ctx.addField(funcInterface, key, getter); - ref = new Reference(key, getterType); - fieldMap.put(key, ref); - } - if (!fieldType.isPrimitive()) { - Expression v = inlineInvoke(ref, methodInfo.f1, OBJECT_TYPE, fieldNullable, inputBeanExpr); - return tryCastIfPublic(v, descriptor.getTypeRef(), fieldName); - } else { - return new Invoke(ref, methodInfo.f1, fieldType, fieldNullable, inputBeanExpr); - } - } - } - - /** Returns an expression that get field value> from bean using reflection. */ - private Expression reflectAccessField( - Expression inputObject, Class cls, Descriptor descriptor) { - Reference fieldRef = getReflectField(cls, descriptor.getField()); - // boolean fieldNullable = !descriptor.getTypeToken().isPrimitive(); - Invoke getObj = - new Invoke(fieldRef, "get", OBJECT_TYPE, fieldNullable(descriptor), inputObject); - return new Cast(getObj, descriptor.getTypeRef(), descriptor.getName()); - } - - /** Returns an expression that get field value> from bean using `Unsafe`. */ - private Expression unsafeAccessField( - Expression inputObject, Class cls, Descriptor descriptor) { - String fieldName = descriptor.getName(); - Reference fieldAccessor = getFieldAccessor(cls, descriptor); - boolean fieldNullable = fieldNullable(descriptor); - if (descriptor.getTypeRef().isPrimitive()) { - Preconditions.checkArgument(!fieldNullable); - TypeRef returnType = descriptor.getTypeRef(); - String funcName = "get" + StringUtils.capitalize(descriptor.getRawType().toString()); - return new Invoke(fieldAccessor, funcName, returnType, false, inputObject); - } else { - Invoke getObj = new Invoke(fieldAccessor, "getObject", OBJECT_TYPE, fieldNullable, inputObject); - return tryCastIfPublic(getObj, descriptor.getTypeRef(), fieldName); - } - } - - private Reference getFieldAccessor(Class cls, Descriptor descriptor) { - Field field = descriptor.getField(); - String fieldName = descriptor.getName(); - String fieldAccessorName = - (duplicatedFields.contains(fieldName) - ? field.getDeclaringClass().getName().replaceAll("\\.|\\$", "_") + "_" - : "") - + fieldName - + "_accessor_"; - return getOrCreateField( - true, - FieldAccessor.class, - fieldAccessorName, - () -> - new StaticInvoke( - FieldAccessor.class, - "createAccessor", - TypeRef.of(FieldAccessor.class), - getReflectField(field.getDeclaringClass(), field, false))); - } - - /** - * Returns an expression that deserialize data as a java bean of type {@link - * CodecBuilder#beanClass}. - */ - public abstract Expression buildDecodeExpression(); - - /** Returns an expression that set field value to bean. */ - protected Expression setFieldValue(Expression bean, Descriptor d, Expression value) { - String fieldName = d.getName(); - if (value instanceof Inlineable) { - ((Inlineable) value).inline(); - } - if (duplicatedFields.contains(fieldName) || !sourcePublicAccessible(beanClass)) { - return unsafeSetField(bean, d, value); - } - if (!d.isFinalField() - && Modifier.isPublic(d.getModifiers()) - && Modifier.isPublic(d.getRawType().getModifiers())) { - if (!d.getRawType().isAssignableFrom(value.type().getRawType())) { - value = tryInlineCast(value, d.getTypeRef()); - } - return new Expression.SetField(bean, fieldName, value); - } else if (d.getWriteMethod() != null && Modifier.isPublic(d.getWriteMethod().getModifiers())) { - if (!d.getRawType().isAssignableFrom(value.type().getRawType())) { - value = tryInlineCast(value, d.getTypeRef()); - } - return new Invoke(bean, d.getWriteMethod().getName(), value); - } else { - if (!d.isFinalField() && !Modifier.isPrivate(d.getModifiers())) { - if (AccessorHelper.defineSetter(d.getField())) { - Class accessorClass = AccessorHelper.getAccessorClass(d.getField()); - if (!d.getRawType().isAssignableFrom(value.type().getRawType())) { - value = tryInlineCast(value, d.getTypeRef()); - } - return new StaticInvoke( - accessorClass, d.getName(), PRIMITIVE_VOID_TYPE, false, bean, value); - } - } - if (d.getWriteMethod() != null && !Modifier.isPrivate(d.getWriteMethod().getModifiers())) { - if (AccessorHelper.defineSetter(d.getWriteMethod())) { - Class accessorClass = AccessorHelper.getAccessorClass(d.getWriteMethod()); - if (!d.getRawType().isAssignableFrom(value.type().getRawType())) { - value = tryInlineCast(value, d.getTypeRef()); - } - return new StaticInvoke( - accessorClass, d.getWriteMethod().getName(), PRIMITIVE_VOID_TYPE, false, bean, value); - } - } - return unsafeSetField(bean, d, value); - } - } - - /** - * Returns an expression that set field value to bean using reflection. - */ - private Expression reflectSetField(Expression bean, Field field, Expression value) { - // Class maybe have getter, but don't have setter, so we can't rely on reflectAccessField to - // populate fieldMap - Reference fieldRef = getReflectField(getRawType(bean.type()), field); - Preconditions.checkNotNull(fieldRef); - return new Invoke(fieldRef, "set", bean, value); - } - - /** - * Returns an expression that set field value to bean using `Unsafe`. - */ - private Expression unsafeSetField(Expression bean, Descriptor descriptor, Expression value) { - TypeRef fieldType = descriptor.getTypeRef(); - Reference fieldAccessor = getFieldAccessor(beanClass, descriptor); - if (descriptor.getTypeRef().isPrimitive()) { - Preconditions.checkArgument(getRawType(value.type()) == getRawType(fieldType)); - String funcName = "put" + StringUtils.capitalize(getRawType(fieldType).toString()); - return new Invoke(fieldAccessor, funcName, bean, value); - } else { - return new Invoke(fieldAccessor, "putObject", bean, value); - } - } - - private Reference getReflectField(Class cls, Field field) { - return getReflectField(cls, field, true); - } - - private Reference getReflectField(Class cls, Field field, boolean setAccessible) { - String fieldName = field.getName(); - String fieldRefName; - if (duplicatedFields.contains(fieldName)) { - fieldRefName = - field.getDeclaringClass().getName().replaceAll("\\.|\\$", "_") + "_" + fieldName + "_Field"; - } else { - fieldRefName = fieldName + "_Field"; - } - return getOrCreateField( - true, - Field.class, - fieldRefName, - () -> { - TypeRef fieldTypeRef = TypeRef.of(Field.class); - Class declaringClass = field.getDeclaringClass(); - Expression classExpr = - staticClassFieldExpr( - declaringClass, declaringClass.getName().replaceAll("\\.|\\$", "_") + "__class__"); - Expression fieldExpr; - if (GraalvmSupport.isGraalBuildTime()) { - fieldExpr = - inlineInvoke( - classExpr, "getDeclaredField", fieldTypeRef, Literal.ofString(fieldName)); - } else { - fieldExpr = - reflectionUtilsInvoke( - "getField", fieldTypeRef, classExpr, Literal.ofString(fieldName)); - } - if (!setAccessible) { - return fieldExpr; - } - Invoke setAccess = new Invoke(fieldExpr, "setAccessible", Literal.True); - return new ListExpression(setAccess, fieldExpr); - }); - } - - protected Reference getOrCreateField( - boolean isStatic, Class type, String fieldName, Supplier value) { - Reference fieldRef = fieldMap.get(fieldName); - if (fieldRef == null) { - fieldName = ctx.newName(fieldName); - ctx.addField(isStatic, true, ctx.type(type), fieldName, value.get()); - fieldRef = new Reference(fieldName, TypeRef.of(type)); - fieldMap.put(fieldName, fieldRef); - } - return fieldRef; - } - - /** Returns an Expression that create a new java object of type {@link CodecBuilder#beanClass}. */ - protected Expression newBean() { - // TODO allow default access-level class. - if (sourcePublicAccessible(beanClass) && ReflectionUtils.hasPublicNoArgConstructor(beanClass)) { - return new Expression.NewInstance(beanType); - } else { - ObjectCreators.getObjectCreator(beanClass); // trigger cache - Invoke newInstance = new Invoke(getObjectCreator(beanClass), "newInstance", OBJECT_TYPE); - return sourcePublicAccessible(beanClass) ? new Cast(newInstance, beanType) : newInstance; - } - } - - protected Expression getObjectCreator(Class type) { - ObjectCreators.getObjectCreator(type); // trigger cache - return getOrCreateField( - true, - ObjectCreator.class, - ctx.newName("objectCreator_" + type.getSimpleName()), - () -> - new StaticInvoke( - ObjectCreators.class, - "getObjectCreator", - TypeRef.of(ObjectCreator.class), - staticBeanClassExpr())); - } - - protected void buildRecordComponentDefaultValues() { - ctx.reserveName(recordComponentDefaultValues.name()); - StaticInvoke expr = - new StaticInvoke( - RecordUtils.class, - "buildRecordComponentDefaultValues", - OBJECT_ARRAY_TYPE, - beanClassExpr()); - ctx.addField(Object[].class, recordComponentDefaultValues.name(), expr); - } - - static boolean recordCtrAccessible(Class cls) { - // support unexported packages in module - if (!Modifier.isPublic(cls.getModifiers())) { - return false; - } - for (RecordComponent component : Objects.requireNonNull(RecordUtils.getRecordComponents(cls))) { - if (!Modifier.isPublic(component.getType().getModifiers())) { - return false; - } - } - return true; - } - - protected Expression beanClassExpr(Class cls) { - if (cls == beanClass) { - return staticBeanClassExpr(); - } - if (GraalvmSupport.isGraalBuildTime()) { - String name = cls.getName().replaceAll("\\.|\\$", "_") + "__class__"; - return getOrCreateField( - true, - Class.class, - name, - () -> - inlineReflectionUtilsInvoke( - "loadClass", CLASS_TYPE, Literal.ofString(cls.getName()))); - } - throw new UnsupportedOperationException(); - } - - protected Expression beanClassExpr() { - if (GraalvmSupport.isGraalBuildTime()) { - return staticBeanClassExpr(); - } - throw new UnsupportedOperationException(); - } - - protected Expression staticBeanClassExpr() { - if (sourcePublicAccessible(beanClass)) { - return Literal.ofClass(beanClass); - } - return staticClassFieldExpr(beanClass, "__class__"); - } - - protected Expression staticClassFieldExpr(Class cls, String fieldName) { - if (sourcePublicAccessible(cls)) { - return Literal.ofClass(cls); - } - return getOrCreateField( - true, - Class.class, - fieldName, - () -> - inlineReflectionUtilsInvoke("loadClass", CLASS_TYPE, Literal.ofString(cls.getName()))); - } - - private StaticInvoke reflectionUtilsInvoke( - String methodName, TypeRef returnType, Expression... arguments) { - return new StaticInvoke( - ReflectionUtils.class, methodName, "", returnType, false, false, false, arguments); - } - - private StaticInvoke inlineReflectionUtilsInvoke( - String methodName, TypeRef returnType, Expression... arguments) { - return new StaticInvoke( - ReflectionUtils.class, methodName, "", returnType, false, true, false, arguments); - } - - /** Build unsafePut operation. */ - protected Expression unsafePut(Expression base, Expression pos, Expression value) { - return new StaticInvoke(UnsafeOps.class, "putByte", base, pos, value); - } - - protected Expression unsafePutBoolean(Expression base, Expression pos, Expression value) { - return new StaticInvoke(UnsafeOps.class, "putBoolean", base, pos, value); - } - - protected Expression unsafePutChar(Expression base, Expression pos, Expression value) { - return new StaticInvoke(UnsafeOps.class, "putChar", base, pos, value); - } - - protected Expression unsafePutShort(Expression base, Expression pos, Expression value) { - return new StaticInvoke(UnsafeOps.class, "putShort", base, pos, value); - } - - protected Expression unsafePutInt(Expression base, Expression pos, Expression value) { - return new StaticInvoke(UnsafeOps.class, "putInt", base, pos, value); - } - - protected Expression unsafePutLong(Expression base, Expression pos, Expression value) { - return new StaticInvoke(UnsafeOps.class, "putLong", base, pos, value); - } - - protected Expression unsafePutFloat(Expression base, Expression pos, Expression value) { - return new StaticInvoke(UnsafeOps.class, "putFloat", base, pos, value); - } - - /** Build unsafePutDouble operation. */ - protected Expression unsafePutDouble(Expression base, Expression pos, Expression value) { - return new StaticInvoke(UnsafeOps.class, "putDouble", base, pos, value); - } - - /** Build unsafeGet operation. */ - protected Expression unsafeGet(Expression base, Expression pos) { - return new StaticInvoke(UnsafeOps.class, "getByte", PRIMITIVE_BYTE_TYPE, base, pos); - } - - protected Expression unsafeGetBoolean(Expression base, Expression pos) { - return new StaticInvoke(UnsafeOps.class, "getBoolean", PRIMITIVE_BOOLEAN_TYPE, base, pos); - } - - protected Expression unsafeGetChar(Expression base, Expression pos) { - StaticInvoke expr = - new StaticInvoke(UnsafeOps.class, "getChar", PRIMITIVE_CHAR_TYPE, base, pos); - if (!NativeByteOrder.IS_LITTLE_ENDIAN) { - expr = new StaticInvoke(Character.class, "reverseBytes", PRIMITIVE_CHAR_TYPE, expr.inline()); - } - return expr; - } - - protected Expression unsafeGetShort(Expression base, Expression pos) { - StaticInvoke expr = - new StaticInvoke(UnsafeOps.class, "getShort", PRIMITIVE_SHORT_TYPE, base, pos); - if (!NativeByteOrder.IS_LITTLE_ENDIAN) { - expr = new StaticInvoke(Short.class, "reverseBytes", PRIMITIVE_SHORT_TYPE, expr.inline()); - } - return expr; - } - - protected Expression unsafeGetInt(Expression base, Expression pos) { - StaticInvoke expr = new StaticInvoke(UnsafeOps.class, "getInt", PRIMITIVE_INT_TYPE, base, pos); - if (!NativeByteOrder.IS_LITTLE_ENDIAN) { - expr = new StaticInvoke(Integer.class, "reverseBytes", PRIMITIVE_INT_TYPE, expr.inline()); - } - return expr; - } - - protected Expression unsafeGetLong(Expression base, Expression pos) { - StaticInvoke expr = - new StaticInvoke(UnsafeOps.class, "getLong", PRIMITIVE_LONG_TYPE, base, pos); - if (!NativeByteOrder.IS_LITTLE_ENDIAN) { - expr = new StaticInvoke(Long.class, "reverseBytes", PRIMITIVE_LONG_TYPE, expr.inline()); - } - return expr; - } - - protected Expression unsafeGetFloat(Expression base, Expression pos) { - StaticInvoke expr = new StaticInvoke(UnsafeOps.class, "getInt", PRIMITIVE_INT_TYPE, base, pos); - if (!NativeByteOrder.IS_LITTLE_ENDIAN) { - expr = new StaticInvoke(Integer.class, "reverseBytes", PRIMITIVE_INT_TYPE, expr.inline()); - } - return new StaticInvoke(Float.class, "intBitsToFloat", PRIMITIVE_FLOAT_TYPE, expr.inline()); - } - - protected Expression unsafeGetDouble(Expression base, Expression pos) { - StaticInvoke expr = - new StaticInvoke(UnsafeOps.class, "getLong", PRIMITIVE_LONG_TYPE, base, pos); - if (!NativeByteOrder.IS_LITTLE_ENDIAN) { - expr = new StaticInvoke(Long.class, "reverseBytes", PRIMITIVE_LONG_TYPE, expr.inline()); - } - return new StaticInvoke(Double.class, "longBitsToDouble", PRIMITIVE_DOUBLE_TYPE, expr.inline()); - } - - protected Expression readChar(Expression buffer) { - return new Invoke(buffer, "readChar", PRIMITIVE_CHAR_TYPE); - } - - protected Expression readInt16(Expression buffer) { - String func = NativeByteOrder.IS_LITTLE_ENDIAN ? "_readInt16OnLE" : "_readInt16OnBE"; - return new Invoke(buffer, func, PRIMITIVE_SHORT_TYPE); - } - - protected Expression readInt32(Expression buffer) { - String func = NativeByteOrder.IS_LITTLE_ENDIAN ? "_readInt32OnLE" : "_readInt32OnBE"; - return new Invoke(buffer, func, PRIMITIVE_INT_TYPE); - } - - public static String readIntFunc() { - return NativeByteOrder.IS_LITTLE_ENDIAN ? "_readInt32OnLE" : "_readInt32OnBE"; - } - - protected Expression readVarInt32(Expression buffer) { - String func = NativeByteOrder.IS_LITTLE_ENDIAN ? "_readVarInt32OnLE" : "_readVarInt32OnBE"; - return new Invoke(buffer, func, PRIMITIVE_INT_TYPE); - } - - protected Expression readInt64(Expression buffer) { - return new Invoke(buffer, readLongFunc(), PRIMITIVE_LONG_TYPE); - } - - public static String readLongFunc() { - return NativeByteOrder.IS_LITTLE_ENDIAN ? "_readInt64OnLE" : "_readInt64OnBE"; - } - - public static String readInt16Func() { - return NativeByteOrder.IS_LITTLE_ENDIAN ? "_readInt16OnLE" : "_readInt16OnBE"; - } - - public static String readVarInt32Func() { - return NativeByteOrder.IS_LITTLE_ENDIAN ? "_readVarInt32OnLE" : "_readVarInt32OnBE"; - } - - public static String readFloat32Func() { - return NativeByteOrder.IS_LITTLE_ENDIAN ? "_readFloat32OnLE" : "_readFloat32OnBE"; - } - - public static String readFloat64Func() { - return NativeByteOrder.IS_LITTLE_ENDIAN ? "_readFloat64OnLE" : "_readFloat64OnBE"; - } - - protected Expression readFloat32(Expression buffer) { - return new Invoke(buffer, readFloat32Func(), PRIMITIVE_FLOAT_TYPE); - } - - protected Expression readFloat64(Expression buffer) { - return new Invoke(buffer, readFloat64Func(), PRIMITIVE_DOUBLE_TYPE); - } -} diff --git a/java/fory-core/src/main/java25/org/apache/fory/builder/ObjectCodecBuilder.java b/java/fory-core/src/main/java25/org/apache/fory/builder/ObjectCodecBuilder.java deleted file mode 100644 index b5c5f1ca59..0000000000 --- a/java/fory-core/src/main/java25/org/apache/fory/builder/ObjectCodecBuilder.java +++ /dev/null @@ -1,1465 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.fory.builder; - -import static org.apache.fory.codegen.Code.LiteralValue.FalseLiteral; -import static org.apache.fory.codegen.Expression.Invoke.inlineInvoke; -import static org.apache.fory.codegen.ExpressionUtils.add; -import static org.apache.fory.codegen.ExpressionUtils.cast; -import static org.apache.fory.collection.Collections.ofHashSet; -import static org.apache.fory.type.TypeUtils.OBJECT_ARRAY_TYPE; -import static org.apache.fory.type.TypeUtils.OBJECT_TYPE; -import static org.apache.fory.type.TypeUtils.PRIMITIVE_BOOLEAN_TYPE; -import static org.apache.fory.type.TypeUtils.PRIMITIVE_BYTE_ARRAY_TYPE; -import static org.apache.fory.type.TypeUtils.PRIMITIVE_BYTE_TYPE; -import static org.apache.fory.type.TypeUtils.PRIMITIVE_CHAR_TYPE; -import static org.apache.fory.type.TypeUtils.PRIMITIVE_DOUBLE_TYPE; -import static org.apache.fory.type.TypeUtils.PRIMITIVE_FLOAT_TYPE; -import static org.apache.fory.type.TypeUtils.PRIMITIVE_INT_TYPE; -import static org.apache.fory.type.TypeUtils.PRIMITIVE_LONG_TYPE; -import static org.apache.fory.type.TypeUtils.PRIMITIVE_SHORT_TYPE; -import static org.apache.fory.type.TypeUtils.PRIMITIVE_VOID_TYPE; -import static org.apache.fory.type.TypeUtils.SHORT_TYPE; -import static org.apache.fory.type.TypeUtils.getRawType; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.IdentityHashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.SortedMap; -import java.util.TreeMap; -import org.apache.fory.Fory; -import org.apache.fory.codegen.Code; -import org.apache.fory.codegen.CodegenContext; -import org.apache.fory.codegen.Expression; -import org.apache.fory.codegen.Expression.Cast; -import org.apache.fory.codegen.Expression.Inlineable; -import org.apache.fory.codegen.Expression.Invoke; -import org.apache.fory.codegen.Expression.ListExpression; -import org.apache.fory.codegen.Expression.Literal; -import org.apache.fory.codegen.Expression.NewInstance; -import org.apache.fory.codegen.Expression.Reference; -import org.apache.fory.codegen.Expression.ReplaceStub; -import org.apache.fory.codegen.Expression.StaticInvoke; -import org.apache.fory.codegen.ExpressionVisitor; -import org.apache.fory.logging.Logger; -import org.apache.fory.logging.LoggerFactory; -import org.apache.fory.meta.TypeDef; -import org.apache.fory.platform.UnsafeOps; -import org.apache.fory.reflect.ObjectCreator; -import org.apache.fory.reflect.ObjectCreators; -import org.apache.fory.reflect.TypeRef; -import org.apache.fory.serializer.ObjectSerializer; -import org.apache.fory.serializer.AbstractObjectSerializer; -import org.apache.fory.type.BFloat16; -import org.apache.fory.type.Descriptor; -import org.apache.fory.type.DescriptorGrouper; -import org.apache.fory.type.DispatchId; -import org.apache.fory.type.Float16; -import org.apache.fory.type.TypeUtils; -import org.apache.fory.type.Types; -import org.apache.fory.util.StringUtils; -import org.apache.fory.util.function.SerializableSupplier; -import org.apache.fory.util.record.RecordUtils; - -/** - * Generate sequential read/write code for java serialization to speed up performance. It also - * reduces space overhead introduced by aligning. Codegen only for time-consuming field, others - * delegate to fory. - * - *

In order to improve jit-compile and inline, serialization code should be spilt groups to avoid - * huge/big methods. - * - *

With meta context share enabled and compatible mode, this serializer will take all non-inner - * final types as non-final, so that fory can write class definition when write class info for those - * types. - * - * @see ObjectCodecOptimizer for code stats and split heuristics. - */ -public class ObjectCodecBuilder extends BaseObjectCodecBuilder { - private static final Logger LOG = LoggerFactory.getLogger(ObjectCodecBuilder.class); - - private final Literal classVersionHash; - protected ObjectCodecOptimizer objectCodecOptimizer; - protected Map recordReversedMapping; - protected Map fieldIndexes; - protected int[] constructorFieldIndexes; - protected boolean[] constructorFieldMask; - protected Class[] constructorFieldTypes; - - public ObjectCodecBuilder(Class beanClass, Fory fory) { - super(TypeRef.of(beanClass), fory, Generated.GeneratedObjectSerializer.class); - Collection descriptors; - DescriptorGrouper grouper; - boolean shareMeta = fory.getConfig().isMetaShareEnabled(); - if (shareMeta) { - TypeDef typeDef = typeResolver(r -> r.getTypeDef(beanClass, true)); - descriptors = typeResolver(r -> typeDef.getDescriptors(r, beanClass)); - grouper = typeResolver(r -> r.createDescriptorGrouper(typeDef, beanClass)); - } else { - grouper = typeResolver(r -> r.getFieldDescriptorGrouper(beanClass, true, false)); - descriptors = grouper.getSortedDescriptors(); - } - if (org.apache.fory.util.Utils.DEBUG_OUTPUT_ENABLED) { - LOG.info( - "========== {} sorted descriptors for {} ==========", - descriptors.size(), - beanClass.getSimpleName()); - List sortedDescriptors = grouper.getSortedDescriptors(); - for (Descriptor d : sortedDescriptors) { - LOG.info( - " {} -> {}, ref {}, nullable {}", - StringUtils.toSnakeCase(d.getName()), - d.getTypeName(), - d.isTrackingRef(), - d.isNullable()); - } - } - classVersionHash = - typeResolver.checkClassVersion() - ? new Literal( - ObjectSerializer.computeStructHash(typeResolver, grouper), PRIMITIVE_INT_TYPE) - : null; - objectCodecOptimizer = new ObjectCodecOptimizer(beanClass, grouper, false, ctx); - if (isRecord) { - if (!recordCtrAccessible) { - buildRecordComponentDefaultValues(); - } - recordReversedMapping = RecordUtils.buildFieldToComponentMapping(beanClass); - } else { - initConstructorFields(grouper.getSortedDescriptors(), true); - } - } - - protected ObjectCodecBuilder(TypeRef beanType, Fory fory, Class superSerializerClass) { - super(beanType, fory, superSerializerClass); - this.classVersionHash = null; - if (isRecord) { - if (!recordCtrAccessible) { - buildRecordComponentDefaultValues(); - } - recordReversedMapping = RecordUtils.buildFieldToComponentMapping(beanClass); - } - } - - protected final void initConstructorFields( - List sortedDescriptors, boolean allowMissingNonFinal) { - initConstructorFields(sortedDescriptors, allowMissingNonFinal, null); - } - - protected final void initConstructorFields( - List sortedDescriptors, boolean allowMissingNonFinal, String[] defaultFields) { - initConstructorFields(sortedDescriptors, allowMissingNonFinal, defaultFields, null); - } - - protected final void initConstructorFields( - List sortedDescriptors, - boolean allowMissingNonFinal, - String[] defaultFields, - Class[] defaultDeclaringClasses) { - ObjectCreator objectCreator = ObjectCreators.getObjectCreator(beanClass); - if (!objectCreator.hasConstructorFields()) { - return; - } - fieldIndexes = buildFieldIndexes(sortedDescriptors); - constructorFieldTypes = objectCreator.getConstructorFieldTypes(); - constructorFieldIndexes = - buildConstructorFieldIndexes( - sortedDescriptors, - objectCreator, - allowMissingNonFinal, - defaultFields, - defaultDeclaringClasses); - constructorFieldMask = buildConstructorFieldMask(sortedDescriptors.size()); - } - - @Override - protected String codecSuffix() { - return ""; - } - - @Override - protected void addCommonImports() { - super.addCommonImports(); - ctx.addImport(Generated.GeneratedObjectSerializer.class); - } - - /** - * Return an expression that serialize java bean of type {@link CodecBuilder#beanClass} to buffer. - */ - @Override - public Expression buildEncodeExpression() { - Reference inputObject = new Reference(ROOT_OBJECT_NAME, OBJECT_TYPE, false); - Reference buffer = new Reference(BUFFER_NAME, bufferTypeRef, false); - - ListExpression expressions = new ListExpression(); - Expression bean = tryCastIfPublic(inputObject, beanType, ctx.newName(beanClass)); - expressions.add(bean); - if (typeResolver.checkClassVersion()) { - expressions.add(new Invoke(buffer, "writeInt32", classVersionHash)); - } - expressions.addAll(serializePrimitives(bean, buffer, objectCodecOptimizer.primitiveGroups)); - int numGroups = getNumGroups(objectCodecOptimizer); - addGroupExpressions( - objectCodecOptimizer.boxedWriteGroups, numGroups, expressions, bean, buffer); - addGroupExpressions( - objectCodecOptimizer.nonPrimitiveWriteGroups, numGroups, expressions, bean, buffer); - return expressions; - } - - private void addGroupExpressions( - List> writeGroup, - int numGroups, - ListExpression expressions, - Expression bean, - Reference buffer) { - for (List group : writeGroup) { - if (group.isEmpty()) { - continue; - } - boolean inline = hasFewFields() || (group.size() == 1 && numGroups < 10); - expressions.add(serializeGroup(group, bean, buffer, inline)); - } - } - - protected boolean hasFewFields() { - return objectCodecOptimizer.descriptorGrouper.getNumDescriptors() < 6; - } - - protected int getNumGroups(ObjectCodecOptimizer objectCodecOptimizer) { - return objectCodecOptimizer.boxedWriteGroups.size() - + objectCodecOptimizer.nonPrimitiveWriteGroups.size(); - } - - private static Map buildFieldIndexes(List descriptors) { - Map indexes = new IdentityHashMap<>(); - for (int i = 0; i < descriptors.size(); i++) { - indexes.put(descriptors.get(i), i); - } - return indexes; - } - - private int[] buildConstructorFieldIndexes( - List descriptors, - ObjectCreator objectCreator, - boolean allowMissingNonFinal, - String[] defaultFields, - Class[] defaultDeclaringClasses) { - String[] names = objectCreator.getConstructorFieldNames(); - Class[] declaringClasses = objectCreator.getConstructorFieldDeclaringClasses(); - boolean[] finalFields = objectCreator.getConstructorFieldFinal(); - int[] indexes = new int[names.length]; - for (int i = 0; i < names.length; i++) { - Class declaringClass = declaringClasses == null ? null : declaringClasses[i]; - boolean allowMissing = - (allowMissingNonFinal && !finalFields[i]) - || contains(defaultFields, defaultDeclaringClasses, names[i], declaringClass); - indexes[i] = - constructorFieldIndex(descriptors, declaringClass, names[i], allowMissing); - } - return indexes; - } - - private static boolean contains( - String[] values, Class[] declaringClasses, String value, Class declaringClass) { - if (values == null) { - return false; - } - for (int i = 0; i < values.length; i++) { - if (values[i].equals(value) - && (declaringClasses == null - || i >= declaringClasses.length - || declaringClasses[i] == null - || declaringClasses[i] == declaringClass)) { - return true; - } - } - return false; - } - - private int constructorFieldIndex( - List descriptors, - Class declaringClass, - String fieldName, - boolean allowMissing) { - int index = -1; - for (int i = 0; i < descriptors.size(); i++) { - Descriptor descriptor = descriptors.get(i); - if (!descriptor.getName().equals(fieldName) - || (declaringClass != null - && (descriptor.getField() == null - || descriptor.getField().getDeclaringClass() != declaringClass))) { - continue; - } - if (index >= 0) { - throw new IllegalStateException( - "Constructor field " + fieldName + " is ambiguous for " + beanClass); - } - index = i; - } - if (index < 0) { - if (allowMissing) { - return -1; - } - throw new IllegalStateException( - "Constructor field " + fieldName + " is not serialized for " + beanClass); - } - return index; - } - - private boolean[] buildConstructorFieldMask(int size) { - boolean[] mask = new boolean[size]; - for (int index : constructorFieldIndexes) { - if (index >= 0) { - mask[index] = true; - } - } - return mask; - } - - private Expression serializeGroup( - List group, Expression bean, Expression buffer, boolean inline) { - SerializableSupplier expressionSupplier = - () -> { - ListExpression groupExpressions = new ListExpression(); - for (Descriptor d : group) { - // `bean` will be replaced by `Reference` to cut-off expr dependency. - Expression fieldValue = getFieldValue(bean, d); - walkPath.add(d.getDeclaringClass() + d.getName()); - Expression fieldExpr = serializeField(fieldValue, buffer, d); - walkPath.removeLast(); - groupExpressions.add(fieldExpr); - } - return groupExpressions; - }; - if (inline) { - return expressionSupplier.get(); - } - return objectCodecOptimizer.invokeGenerated( - writeCutPoints(bean, buffer), expressionSupplier.get(), "writeFields"); - } - - /** - * Return a list of expressions that serialize all primitive fields. This can reduce unnecessary - * grow call and increment writerIndex in writeXXX. - */ - private List serializePrimitives( - Expression bean, Expression buffer, List> primitiveGroups) { - int totalSize = getTotalSizeOfPrimitives(primitiveGroups); - if (totalSize == 0) { - return new ArrayList<>(); - } - if (config.compressInt() || config.compressLong()) { - return serializePrimitivesCompressed(bean, buffer, primitiveGroups, totalSize); - } else { - return serializePrimitivesUnCompressed(bean, buffer, primitiveGroups, totalSize); - } - } - - protected int getNumPrimitiveFields(List> primitiveGroups) { - return primitiveGroups.stream().mapToInt(List::size).sum(); - } - - private List serializePrimitivesUnCompressed( - Expression bean, Expression buffer, List> primitiveGroups, int totalSize) { - List expressions = new ArrayList<>(); - int numPrimitiveFields = getNumPrimitiveFields(primitiveGroups); - Literal totalSizeLiteral = new Literal(totalSize, PRIMITIVE_INT_TYPE); - // After this grow, following writes can be unsafe without checks. - expressions.add(new Invoke(buffer, "grow", totalSizeLiteral)); - Expression writerIndex = - new Invoke(buffer, "writerIndex", "writerIndex", PRIMITIVE_INT_TYPE); - expressions.add(writerIndex); - int acc = 0; - for (List group : primitiveGroups) { - ListExpression groupExpressions = new ListExpression(); - // use Reference to cut-off expr dependency. - for (Descriptor descriptor : group) { - int dispatchId = getNumericDescriptorDispatchId(descriptor); - // `bean` will be replaced by `Reference` to cut-off expr dependency. - Expression fieldValue = getFieldValue(bean, descriptor); - if (fieldValue instanceof Inlineable) { - ((Inlineable) fieldValue).inline(); - } - if (dispatchId == DispatchId.BOOL) { - groupExpressions.add(bufferPutBoolean(buffer, getBufferIndex(writerIndex, acc), fieldValue)); - acc += 1; - } else if (dispatchId == DispatchId.INT8) { - groupExpressions.add(bufferPutByte(buffer, getBufferIndex(writerIndex, acc), fieldValue)); - acc += 1; - } else if (dispatchId == DispatchId.UINT8) { - groupExpressions.add( - bufferPutByte( - buffer, getBufferIndex(writerIndex, acc), primitiveByteValue(fieldValue, descriptor))); - acc += 1; - } else if (dispatchId == DispatchId.CHAR) { - groupExpressions.add(bufferPutChar(buffer, getBufferIndex(writerIndex, acc), fieldValue)); - acc += 2; - } else if (dispatchId == DispatchId.INT16) { - groupExpressions.add(bufferPutInt16(buffer, getBufferIndex(writerIndex, acc), fieldValue)); - acc += 2; - } else if (dispatchId == DispatchId.UINT16) { - groupExpressions.add( - bufferPutInt16( - buffer, - getBufferIndex(writerIndex, acc), - primitiveShortValue(fieldValue, descriptor))); - acc += 2; - } else if (dispatchId == DispatchId.FLOAT16 || dispatchId == DispatchId.BFLOAT16) { - groupExpressions.add( - bufferPutInt16( - buffer, - getBufferIndex(writerIndex, acc), - new Invoke(fieldValue, "toBits", SHORT_TYPE))); - acc += 2; - } else if (dispatchId == DispatchId.INT32) { - groupExpressions.add(bufferPutInt32(buffer, getBufferIndex(writerIndex, acc), fieldValue)); - acc += 4; - } else if (dispatchId == DispatchId.UINT32) { - groupExpressions.add( - bufferPutInt32( - buffer, getBufferIndex(writerIndex, acc), primitiveIntValue(fieldValue, descriptor))); - acc += 4; - } else if (dispatchId == DispatchId.INT64 || dispatchId == DispatchId.UINT64) { - groupExpressions.add(bufferPutInt64(buffer, getBufferIndex(writerIndex, acc), fieldValue)); - acc += 8; - } else if (dispatchId == DispatchId.FLOAT32) { - groupExpressions.add(bufferPutFloat32(buffer, getBufferIndex(writerIndex, acc), fieldValue)); - acc += 4; - } else if (dispatchId == DispatchId.FLOAT64) { - groupExpressions.add(bufferPutFloat64(buffer, getBufferIndex(writerIndex, acc), fieldValue)); - acc += 8; - } else { - throw new IllegalStateException("Unsupported dispatchId: " + dispatchId); - } - } - if (hasFewFields() || numPrimitiveFields < 4) { - expressions.add(groupExpressions); - } else { - expressions.add( - objectCodecOptimizer.invokeGenerated( - ofHashSet(bean, buffer, writerIndex), groupExpressions, "writeFields")); - } - } - Expression increaseWriterIndex = - new Invoke( - buffer, - "_increaseWriterIndexUnsafe", - new Literal(totalSizeLiteral, PRIMITIVE_INT_TYPE)); - expressions.add(increaseWriterIndex); - return expressions; - } - - private List serializePrimitivesCompressed( - Expression bean, Expression buffer, List> primitiveGroups, int totalSize) { - List expressions = new ArrayList<>(); - // int/long may need extra one-byte for writing. - int extraSize = 0; - for (List group : primitiveGroups) { - for (Descriptor d : group) { - int id = getNumericDescriptorDispatchId(d); - if (id == DispatchId.INT32 - || id == DispatchId.VARINT32 - || id == DispatchId.VAR_UINT32 - || id == DispatchId.UINT32) { - // varint may be written as 5bytes, use 8bytes for written as long to reduce cost. - extraSize += 4; - } else if (id == DispatchId.INT64 - || id == DispatchId.VARINT64 - || id == DispatchId.TAGGED_INT64 - || id == DispatchId.VAR_UINT64 - || id == DispatchId.TAGGED_UINT64 - || id == DispatchId.UINT64) { - extraSize += 1; // long use 1~9 bytes. - } - } - } - int growSize = totalSize + extraSize; - // After this grow, following writes can be unsafe without checks. - expressions.add(new Invoke(buffer, "grow", Literal.ofInt(growSize))); - int numPrimitiveFields = getNumPrimitiveFields(primitiveGroups); - for (List group : primitiveGroups) { - ListExpression groupExpressions = new ListExpression(); - Expression writerIndex = - new Invoke(buffer, "writerIndex", "writerIndex", PRIMITIVE_INT_TYPE); - // use Reference to cut-off expr dependency. - int acc = 0; - boolean compressStarted = false; - for (Descriptor descriptor : group) { - int dispatchId = getNumericDescriptorDispatchId(descriptor); - // `bean` will be replaced by `Reference` to cut-off expr dependency. - Expression fieldValue = getFieldValue(bean, descriptor); - if (fieldValue instanceof Inlineable) { - ((Inlineable) fieldValue).inline(); - } - if (dispatchId == DispatchId.BOOL) { - groupExpressions.add(bufferPutBoolean(buffer, getBufferIndex(writerIndex, acc), fieldValue)); - acc += 1; - } else if (dispatchId == DispatchId.INT8) { - groupExpressions.add(bufferPutByte(buffer, getBufferIndex(writerIndex, acc), fieldValue)); - acc += 1; - } else if (dispatchId == DispatchId.UINT8) { - groupExpressions.add( - bufferPutByte( - buffer, getBufferIndex(writerIndex, acc), primitiveByteValue(fieldValue, descriptor))); - acc += 1; - } else if (dispatchId == DispatchId.CHAR) { - groupExpressions.add(bufferPutChar(buffer, getBufferIndex(writerIndex, acc), fieldValue)); - acc += 2; - } else if (dispatchId == DispatchId.INT16) { - groupExpressions.add(bufferPutInt16(buffer, getBufferIndex(writerIndex, acc), fieldValue)); - acc += 2; - } else if (dispatchId == DispatchId.UINT16) { - groupExpressions.add( - bufferPutInt16( - buffer, - getBufferIndex(writerIndex, acc), - primitiveShortValue(fieldValue, descriptor))); - acc += 2; - } else if (dispatchId == DispatchId.FLOAT16 || dispatchId == DispatchId.BFLOAT16) { - groupExpressions.add( - bufferPutInt16( - buffer, - getBufferIndex(writerIndex, acc), - new Invoke(fieldValue, "toBits", SHORT_TYPE))); - acc += 2; - } else if (dispatchId == DispatchId.FLOAT32) { - groupExpressions.add(bufferPutFloat32(buffer, getBufferIndex(writerIndex, acc), fieldValue)); - acc += 4; - } else if (dispatchId == DispatchId.FLOAT64) { - groupExpressions.add(bufferPutFloat64(buffer, getBufferIndex(writerIndex, acc), fieldValue)); - acc += 8; - } else if (dispatchId == DispatchId.INT32) { - groupExpressions.add(bufferPutInt32(buffer, getBufferIndex(writerIndex, acc), fieldValue)); - acc += 4; - } else if (dispatchId == DispatchId.UINT32) { - groupExpressions.add( - bufferPutInt32( - buffer, getBufferIndex(writerIndex, acc), primitiveIntValue(fieldValue, descriptor))); - acc += 4; - } else if (dispatchId == DispatchId.INT64 || dispatchId == DispatchId.UINT64) { - groupExpressions.add(bufferPutInt64(buffer, getBufferIndex(writerIndex, acc), fieldValue)); - acc += 8; - } else if (dispatchId == DispatchId.VARINT32) { - if (!compressStarted) { - addIncWriterIndexExpr(groupExpressions, buffer, acc); - compressStarted = true; - } - groupExpressions.add(new Invoke(buffer, "_unsafeWriteVarInt32", fieldValue)); - } else if (dispatchId == DispatchId.VAR_UINT32) { - if (!compressStarted) { - addIncWriterIndexExpr(groupExpressions, buffer, acc); - compressStarted = true; - } - groupExpressions.add( - new Invoke( - buffer, "_unsafeWriteVarUInt32", primitiveIntValue(fieldValue, descriptor))); - } else if (dispatchId == DispatchId.VARINT64) { - if (!compressStarted) { - addIncWriterIndexExpr(groupExpressions, buffer, acc); - compressStarted = true; - } - groupExpressions.add(new Invoke(buffer, "writeVarInt64", fieldValue)); - } else if (dispatchId == DispatchId.TAGGED_INT64) { - if (!compressStarted) { - addIncWriterIndexExpr(groupExpressions, buffer, acc); - compressStarted = true; - } - groupExpressions.add(new Invoke(buffer, "writeTaggedInt64", fieldValue)); - } else if (dispatchId == DispatchId.VAR_UINT64) { - if (!compressStarted) { - addIncWriterIndexExpr(groupExpressions, buffer, acc); - compressStarted = true; - } - groupExpressions.add(new Invoke(buffer, "writeVarUInt64", fieldValue)); - } else if (dispatchId == DispatchId.TAGGED_UINT64) { - if (!compressStarted) { - addIncWriterIndexExpr(groupExpressions, buffer, acc); - compressStarted = true; - } - groupExpressions.add(new Invoke(buffer, "writeTaggedUInt64", fieldValue)); - } else { - throw new IllegalStateException("Unsupported dispatchId: " + dispatchId); - } - } - if (!compressStarted) { - // int/long are sorted in the last. - addIncWriterIndexExpr(groupExpressions, buffer, acc); - } - if (hasFewFields() || numPrimitiveFields < 4) { - expressions.add(groupExpressions); - } else { - expressions.add( - objectCodecOptimizer.invokeGenerated( - ofHashSet(bean, buffer, writerIndex), groupExpressions, "writeFields")); - } - } - return expressions; - } - - private Expression bufferPutByte(Expression buffer, Expression index, Expression value) { - return new Invoke(buffer, "_unsafePutByte", index, value); - } - - private Expression bufferPutBoolean(Expression buffer, Expression index, Expression value) { - return new Invoke(buffer, "_unsafePutBoolean", index, value); - } - - private Expression bufferPutChar(Expression buffer, Expression index, Expression value) { - return new Invoke(buffer, "_unsafePutChar", index, value); - } - - private Expression bufferPutInt16(Expression buffer, Expression index, Expression value) { - return new Invoke(buffer, "_unsafePutInt16", index, value); - } - - private Expression bufferPutInt32(Expression buffer, Expression index, Expression value) { - return new Invoke(buffer, "_unsafePutInt32", index, value); - } - - private Expression bufferPutInt64(Expression buffer, Expression index, Expression value) { - return new Invoke(buffer, "_unsafePutInt64", index, value); - } - - private Expression bufferPutFloat32(Expression buffer, Expression index, Expression value) { - return bufferPutInt32( - buffer, - index, - new StaticInvoke(Float.class, "floatToRawIntBits", PRIMITIVE_INT_TYPE, value)); - } - - private Expression bufferPutFloat64(Expression buffer, Expression index, Expression value) { - return bufferPutInt64( - buffer, - index, - new StaticInvoke(Double.class, "doubleToRawLongBits", PRIMITIVE_LONG_TYPE, value)); - } - - private Expression bufferGetByte(Expression buffer, Expression index) { - return new Invoke(buffer, "_unsafeGetByte", PRIMITIVE_BYTE_TYPE, index); - } - - private Expression bufferGetBoolean(Expression buffer, Expression index) { - return new Invoke(buffer, "_unsafeGetBoolean", PRIMITIVE_BOOLEAN_TYPE, index); - } - - private Expression bufferGetChar(Expression buffer, Expression index) { - return new Invoke(buffer, "_unsafeGetChar", PRIMITIVE_CHAR_TYPE, index); - } - - private Expression bufferGetInt16(Expression buffer, Expression index) { - return new Invoke(buffer, "_unsafeGetInt16", PRIMITIVE_SHORT_TYPE, index); - } - - private Expression bufferGetInt32(Expression buffer, Expression index) { - return new Invoke(buffer, "_unsafeGetInt32", PRIMITIVE_INT_TYPE, index); - } - - private Expression bufferGetInt64(Expression buffer, Expression index) { - return new Invoke(buffer, "_unsafeGetInt64", PRIMITIVE_LONG_TYPE, index); - } - - private Expression bufferGetFloat32(Expression buffer, Expression index) { - return new StaticInvoke( - Float.class, "intBitsToFloat", PRIMITIVE_FLOAT_TYPE, bufferGetInt32(buffer, index)); - } - - private Expression bufferGetFloat64(Expression buffer, Expression index) { - return new StaticInvoke( - Double.class, - "longBitsToDouble", - PRIMITIVE_DOUBLE_TYPE, - bufferGetInt64(buffer, index)); - } - - private Expression primitiveByteValue(Expression fieldValue, Descriptor descriptor) { - return fieldValue.type().isPrimitive() - ? cast(fieldValue, PRIMITIVE_BYTE_TYPE) - : new Invoke(boxedNumericValue(fieldValue, descriptor), "byteValue", PRIMITIVE_BYTE_TYPE); - } - - private Expression primitiveShortValue(Expression fieldValue, Descriptor descriptor) { - return fieldValue.type().isPrimitive() - ? cast(fieldValue, PRIMITIVE_SHORT_TYPE) - : new Invoke(boxedNumericValue(fieldValue, descriptor), "shortValue", PRIMITIVE_SHORT_TYPE); - } - - private Expression primitiveIntValue(Expression fieldValue, Descriptor descriptor) { - return fieldValue.type().isPrimitive() - ? cast(fieldValue, PRIMITIVE_INT_TYPE) - : new Invoke(boxedNumericValue(fieldValue, descriptor), "intValue", PRIMITIVE_INT_TYPE); - } - - private Expression boxedNumericValue(Expression fieldValue, Descriptor descriptor) { - return Number.class.isAssignableFrom(getRawType(fieldValue.type())) - ? fieldValue - : cast(fieldValue, descriptor.getTypeRef()); - } - - private void addIncWriterIndexExpr(ListExpression expressions, Expression buffer, int diff) { - if (diff != 0) { - expressions.add(new Invoke(buffer, "_increaseWriterIndexUnsafe", Literal.ofInt(diff))); - } - } - - private int getTotalSizeOfPrimitives(List> primitiveGroups) { - return primitiveGroups.stream() - .flatMap(Collection::stream) - .mapToInt( - d -> { - Class rawType = d.getRawType(); - if (TypeUtils.isPrimitive(rawType) || TypeUtils.isBoxed(rawType)) { - return TypeUtils.getSizeOfPrimitiveType(TypeUtils.unwrap(rawType)); - } - return Types.getPrimitiveTypeSize(Types.getDescriptorTypeId(typeResolver, d)); - }) - .sum(); - } - - private Expression getWriterPos(Expression writerPos, long acc) { - if (acc == 0) { - return writerPos; - } - return add(writerPos, Literal.ofLong(acc)); - } - - public Expression buildDecodeExpression() { - Reference buffer = new Reference(BUFFER_NAME, bufferTypeRef, false); - ListExpression expressions = new ListExpression(); - if (typeResolver.checkClassVersion()) { - expressions.add(checkClassVersion(buffer)); - } - if (!isRecord && constructorFieldIndexes != null) { - return buildConstructorDecodeExpression(buffer, expressions); - } - Expression bean; - if (!isRecord) { - if (constructorFieldIndexes == null) { - bean = newBean(); - Expression referenceObject = invokeReadContext("reference", bean); - expressions.add(bean); - expressions.add(referenceObject); - } else { - bean = new FieldsArray(fieldIndexes.size()); - expressions.add(bean); - } - } else { - if (recordCtrAccessible) { - bean = new FieldsCollector(); - } else { - bean = buildComponentsArray(); - } - } - expressions.addAll(deserializePrimitives(bean, buffer, objectCodecOptimizer.primitiveGroups)); - int numGroups = getNumGroups(objectCodecOptimizer); - deserializeReadGroup( - objectCodecOptimizer.boxedReadGroups, numGroups, expressions, bean, buffer); - deserializeReadGroup( - objectCodecOptimizer.nonPrimitiveReadGroups, numGroups, expressions, bean, buffer); - if (isRecord) { - if (recordCtrAccessible) { - assert bean instanceof FieldsCollector; - FieldsCollector collector = (FieldsCollector) bean; - bean = createRecord(collector.recordValuesMap); - } else { - ObjectCreators.getObjectCreator(beanClass); // trigger cache and make error raised early - bean = - new Invoke(getObjectCreator(beanClass), "newInstanceWithArguments", OBJECT_TYPE, bean); - } - } - expressions.add(new Expression.Return(bean)); - return expressions; - } - - private Expression buildConstructorDecodeExpression( - Reference buffer, ListExpression expressions) { - FieldsArray fieldsArray = new FieldsArray(fieldIndexes.size()); - expressions.add(fieldsArray); - expressions.add( - new StaticInvoke( - AbstractObjectSerializer.class, - "beginConstructorRef", - PRIMITIVE_VOID_TYPE, - readContextRef())); - List bufferedNonConstructorFields = new ArrayList<>(); - int remainingConstructorFields = countConstructorFields(); - Expression bean = null; - if (remainingConstructorFields == 0) { - bean = createCtorBean(expressions, fieldsArray); - } - for (Descriptor descriptor : protocolDescriptors()) { - int index = fieldIndexes.get(descriptor); - walkPath.add(descriptor.getDeclaringClass() + descriptor.getName()); - if (constructorFieldMask[index]) { - expressions.add(deserializeToFieldsArray(fieldsArray, buffer, descriptor, true)); - remainingConstructorFields--; - if (remainingConstructorFields == 0) { - bean = createCtorBean(expressions, fieldsArray); - addBufferedFieldSetters(expressions, bean, fieldsArray, bufferedNonConstructorFields); - } - } else if (bean == null) { - expressions.add(deserializeToFieldsArray(fieldsArray, buffer, descriptor, false)); - bufferedNonConstructorFields.add(descriptor); - } else { - expressions.add(deserializeToBean(bean, buffer, descriptor)); - } - walkPath.removeLast(); - } - expressions.add( - new StaticInvoke( - AbstractObjectSerializer.class, - "endConstructorRef", - PRIMITIVE_VOID_TYPE, - readContextRef())); - expressions.add(new Expression.Return(bean)); - return expressions; - } - - private int countConstructorFields() { - int count = 0; - for (boolean constructorField : constructorFieldMask) { - if (constructorField) { - count++; - } - } - return count; - } - - private List protocolDescriptors() { - List descriptors = new ArrayList<>(); - addDescriptors(descriptors, objectCodecOptimizer.primitiveGroups); - addDescriptors(descriptors, objectCodecOptimizer.boxedReadGroups); - addDescriptors(descriptors, objectCodecOptimizer.nonPrimitiveReadGroups); - return descriptors; - } - - private void addDescriptors(List descriptors, List> groups) { - for (List group : groups) { - descriptors.addAll(group); - } - } - - private Expression createCtorBean(ListExpression expressions, FieldsArray fieldsArray) { - Expression bean = createConstructorObject(fieldsArray); - expressions.add( - new StaticInvoke( - AbstractObjectSerializer.class, - "checkNoUnresolvedReadRef", - PRIMITIVE_VOID_TYPE, - readContextRef(), - staticBeanClassExpr())); - expressions.add(bean); - expressions.add( - new StaticInvoke( - AbstractObjectSerializer.class, - "referenceConstructorRef", - PRIMITIVE_VOID_TYPE, - readContextRef(), - bean)); - postCreateConstructorObject(expressions, bean); - return bean; - } - - private Expression deserializeToFieldsArray( - FieldsArray fieldsArray, Reference buffer, Descriptor descriptor, boolean constructorField) { - TypeRef castTypeRef = - hasCompatibleCollectionArrayRead(descriptor) - ? compatibleReadTargetTypeRef(descriptor) - : descriptor.getTypeRef(); - return deserializeField( - buffer, - descriptor, - expr -> { - Expression value = - constructorField ? tryInlineCast(expr, castTypeRef) : new Cast(expr, OBJECT_TYPE); - value = - new StaticInvoke( - AbstractObjectSerializer.class, - constructorField ? "ctorFieldValue" : "bufferFieldValue", - OBJECT_TYPE, - readContextRef(), - value, - staticBeanClassExpr()); - return setFieldValue(fieldsArray, descriptor, value); - }); - } - - private Expression deserializeToBean(Expression bean, Reference buffer, Descriptor descriptor) { - TypeRef castTypeRef = - hasCompatibleCollectionArrayRead(descriptor) - ? compatibleReadTargetTypeRef(descriptor) - : descriptor.getTypeRef(); - return deserializeField( - buffer, descriptor, expr -> setFieldValue(bean, descriptor, tryInlineCast(expr, castTypeRef))); - } - - protected void postCreateConstructorObject(ListExpression expressions, Expression bean) {} - - protected void deserializeReadGroup( - List> readGroups, - int numGroups, - ListExpression expressions, - Expression bean, - Reference buffer) { - for (List group : readGroups) { - if (group.isEmpty()) { - continue; - } - boolean inline = hasFewFields() || (group.size() == 1 && numGroups < 10); - expressions.add(deserializeGroup(group, bean, buffer, inline)); - } - } - - protected Expression buildComponentsArray() { - return new StaticInvoke( - UnsafeOps.class, "copyObjectArray", OBJECT_ARRAY_TYPE, recordComponentDefaultValues); - } - - protected Expression createRecord(SortedMap recordComponents) { - Expression[] params = recordComponents.values().toArray(new Expression[0]); - return new NewInstance(beanType, params); - } - - protected Expression createConstructorObject(FieldsArray fieldValues) { - Expression[] params = new Expression[constructorFieldIndexes.length]; - Expression[] directParams = new Expression[constructorFieldIndexes.length]; - for (int i = 0; i < constructorFieldIndexes.length; i++) { - int index = constructorFieldIndexes[i]; - if (index < 0) { - params[i] = defaultConstructorValue(i); - } else { - params[i] = fieldValue(fieldValues, index); - } - directParams[i] = tryInlineCast(params[i], TypeRef.of(constructorFieldTypes[i])); - } - ObjectCreator objectCreator = ObjectCreators.getObjectCreator(beanClass); - if (objectCreator.isOnlyPublicConstructor() - && sourcePublicAccessible(beanClass) - && constructorParamsAccessible()) { - return new NewInstance(beanType, directParams); - } - Expression args = new Expression.NewArray(OBJECT_ARRAY_TYPE, params); - Expression newInstance = - new Invoke(getObjectCreator(beanClass), "newInstanceWithArguments", OBJECT_TYPE, args); - return sourcePublicAccessible(beanClass) ? new Cast(newInstance, beanType) : newInstance; - } - - protected Expression defaultConstructorValue(int constructorParameterIndex) { - return new StaticInvoke( - AbstractObjectSerializer.class, - "defaultConstructorValue", - OBJECT_TYPE, - staticClassFieldExpr( - constructorFieldTypes[constructorParameterIndex], - "constructorFieldClass" + constructorParameterIndex + "_")); - } - - private boolean constructorParamsAccessible() { - for (Class constructorFieldType : constructorFieldTypes) { - if (!sourcePublicAccessible(constructorFieldType)) { - return false; - } - } - return true; - } - - private void addNonConstructorFieldSetters( - ListExpression expressions, Expression bean, FieldsArray fieldValues) { - for (Descriptor descriptor : objectCodecOptimizer.descriptorGrouper.getSortedDescriptors()) { - int index = fieldIndexes.get(descriptor); - if (constructorFieldMask[index]) { - continue; - } - TypeRef castTypeRef = - hasCompatibleCollectionArrayRead(descriptor) - ? compatibleReadTargetTypeRef(descriptor) - : descriptor.getTypeRef(); - Expression value = - new StaticInvoke( - AbstractObjectSerializer.class, - "resolveBufferedValue", - OBJECT_TYPE, - fieldValue(fieldValues, index), - bean); - value = tryInlineCast(value, castTypeRef); - expressions.add(setFieldValue(bean, descriptor, value)); - } - } - - private void addBufferedFieldSetters( - ListExpression expressions, - Expression bean, - FieldsArray fieldValues, - List descriptors) { - for (Descriptor descriptor : descriptors) { - int index = fieldIndexes.get(descriptor); - TypeRef castTypeRef = - hasCompatibleCollectionArrayRead(descriptor) - ? compatibleReadTargetTypeRef(descriptor) - : descriptor.getTypeRef(); - Expression value = - new StaticInvoke( - AbstractObjectSerializer.class, - "resolveBufferedValue", - OBJECT_TYPE, - fieldValue(fieldValues, index), - bean); - value = tryInlineCast(value, castTypeRef); - expressions.add(setFieldValue(bean, descriptor, value)); - } - } - - private Expression fieldValue(Expression fieldValues, int index) { - return new StaticInvoke( - AbstractObjectSerializer.class, - "fieldValue", - OBJECT_TYPE, - fieldValues, - Literal.ofInt(index)); - } - - private class FieldsCollector extends Expression.AbstractExpression { - private final TreeMap recordValuesMap = new TreeMap<>(); - - protected FieldsCollector() { - super(new Expression[0]); - } - - @Override - public TypeRef type() { - return beanType; - } - - @Override - public Code.ExprCode doGenCode(CodegenContext ctx) { - return new Code.ExprCode(FalseLiteral, Code.variable(getRawType(beanType), "null")); - } - } - - protected class FieldsArray extends Expression.AbstractExpression { - private final int size; - private final String name; - - protected FieldsArray(int size) { - super(new Expression[0]); - this.size = size; - name = ctx.newName("fieldValues"); - } - - @Override - public TypeRef type() { - return OBJECT_ARRAY_TYPE; - } - - @Override - public Code.ExprCode doGenCode(CodegenContext ctx) { - String code = ctx.type(Object[].class) + " " + name + " = new Object[" + size + "];"; - return new Code.ExprCode(code, FalseLiteral, Code.variable(Object[].class, name)); - } - - int fieldIndex(Descriptor descriptor) { - return fieldIndexes.get(descriptor); - } - } - - @Override - protected Expression setFieldValue(Expression bean, Descriptor d, Expression value) { - if (bean instanceof FieldsArray) { - return new Expression.AssignArrayElem( - bean, value, Literal.ofInt(((FieldsArray) bean).fieldIndex(d))); - } - if (isRecord) { - if (recordCtrAccessible) { - if (value instanceof Inlineable) { - ((Inlineable) value).inline(false); - } - int index = recordReversedMapping.get(d.getName()); - FieldsCollector collector = (FieldsCollector) bean; - collector.recordValuesMap.put(index, value); - return value; - } else { - int index = recordReversedMapping.get(d.getName()); - return new Expression.AssignArrayElem(bean, value, Literal.ofInt(index)); - } - } - return super.setFieldValue(bean, d, value); - } - - protected Expression deserializeGroup( - List group, Expression bean, Expression buffer, boolean inline) { - if (isRecord) { - return deserializeGroupForRecord(group, bean, buffer); - } - SerializableSupplier exprSupplier = - () -> { - ListExpression groupExpressions = new ListExpression(); - // use Reference to cut-off expr dependency. - for (Descriptor d : group) { - ExpressionVisitor.ExprHolder exprHolder = ExpressionVisitor.ExprHolder.of("bean", bean); - walkPath.add(d.getDeclaringClass() + d.getName()); - TypeRef castTypeRef = - hasCompatibleCollectionArrayRead(d) - ? compatibleReadTargetTypeRef(d) - : d.getTypeRef(); - Expression action = - deserializeField( - buffer, - d, - // `bean` will be replaced by `Reference` to cut-off expr - // dependency. - expr -> - setFieldValue(exprHolder.get("bean"), d, tryInlineCast(expr, castTypeRef))); - walkPath.removeLast(); - if (needsGeneratedReadFieldMethod(d)) { - action = - objectCodecOptimizer.invokeGenerated( - readCutPoints(bean, buffer), action, "readField"); - } - groupExpressions.add(action); - } - return groupExpressions; - }; - if (inline) { - return exprSupplier.get(); - } else { - return objectCodecOptimizer.invokeGenerated( - readCutPoints(bean, buffer), exprSupplier.get(), "readFields"); - } - } - - private boolean needsGeneratedReadFieldMethod(Descriptor descriptor) { - return !hasFewFields() - && !isMonomorphic(descriptor) - && !useCollectionSerialization(descriptor) - && !useMapSerialization(descriptor.getTypeRef()); - } - - protected Expression deserializeGroupForRecord( - List group, Expression bean, Expression buffer) { - ListExpression groupExpressions = new ListExpression(); - // use Reference to cut-off expr dependency. - for (Descriptor d : group) { - TypeRef castTypeRef = - hasCompatibleCollectionArrayRead(d) ? compatibleReadTargetTypeRef(d) : d.getTypeRef(); - Expression value = deserializeField(buffer, d, expr -> expr); - Expression action = setFieldValue(bean, d, tryInlineCast(value, castTypeRef)); - groupExpressions.add(action); - } - return groupExpressions; - } - - private Expression checkClassVersion(Expression buffer) { - return new StaticInvoke( - ObjectSerializer.class, - "checkClassVersion", - PRIMITIVE_VOID_TYPE, - false, - beanClassExpr(), - inlineInvoke(buffer, readIntFunc(), PRIMITIVE_INT_TYPE), - Objects.requireNonNull(classVersionHash)); - } - - /** - * Return a list of expressions that deserialize all primitive fields. This can reduce unnecessary - * check call and increment readerIndex in writeXXX. - */ - protected List deserializePrimitives( - Expression bean, Expression buffer, List> primitiveGroups) { - int totalSize = getTotalSizeOfPrimitives(primitiveGroups); - if (totalSize == 0) { - return new ArrayList<>(); - } - if (config.compressInt() || config.compressLong()) { - return deserializeCompressedPrimitives(bean, buffer, primitiveGroups); - } else { - return deserializeUnCompressedPrimitives(bean, buffer, primitiveGroups, totalSize); - } - } - - private List deserializeUnCompressedPrimitives( - Expression bean, Expression buffer, List> primitiveGroups, int totalSize) { - List expressions = new ArrayList<>(); - int numPrimitiveFields = getNumPrimitiveFields(primitiveGroups); - Literal totalSizeLiteral = Literal.ofInt(totalSize); - // After this check, following read can be totally unsafe without checks - expressions.add(new Invoke(buffer, "checkReadableBytes", totalSizeLiteral)); - Expression readerIndex = - new Invoke(buffer, "readerIndex", "readerIndex", PRIMITIVE_INT_TYPE); - expressions.add(readerIndex); - int acc = 0; - for (List group : primitiveGroups) { - ListExpression groupExpressions = new ListExpression(); - for (Descriptor descriptor : group) { - int dispatchId = getNumericDescriptorDispatchId(descriptor); - Expression fieldValue; - if (dispatchId == DispatchId.BOOL) { - fieldValue = bufferGetBoolean(buffer, getBufferIndex(readerIndex, acc)); - acc += 1; - } else if (dispatchId == DispatchId.INT8) { - fieldValue = bufferGetByte(buffer, getBufferIndex(readerIndex, acc)); - acc += 1; - } else if (dispatchId == DispatchId.UINT8) { - fieldValue = - new StaticInvoke( - Byte.class, - "toUnsignedInt", - descriptor.getTypeRef(), - bufferGetByte(buffer, getBufferIndex(readerIndex, acc))); - acc += 1; - } else if (dispatchId == DispatchId.CHAR) { - fieldValue = bufferGetChar(buffer, getBufferIndex(readerIndex, acc)); - acc += 2; - } else if (dispatchId == DispatchId.INT16) { - fieldValue = bufferGetInt16(buffer, getBufferIndex(readerIndex, acc)); - acc += 2; - } else if (dispatchId == DispatchId.UINT16) { - fieldValue = - new StaticInvoke( - Short.class, - "toUnsignedInt", - descriptor.getTypeRef(), - bufferGetInt16(buffer, getBufferIndex(readerIndex, acc))); - acc += 2; - } else if (dispatchId == DispatchId.FLOAT16) { - fieldValue = - new StaticInvoke( - Float16.class, - "fromBits", - TypeRef.of(Float16.class), - bufferGetInt16(buffer, getBufferIndex(readerIndex, acc))); - acc += 2; - } else if (dispatchId == DispatchId.BFLOAT16) { - fieldValue = - new StaticInvoke( - BFloat16.class, - "fromBits", - TypeRef.of(BFloat16.class), - bufferGetInt16(buffer, getBufferIndex(readerIndex, acc))); - acc += 2; - } else if (dispatchId == DispatchId.INT32) { - fieldValue = bufferGetInt32(buffer, getBufferIndex(readerIndex, acc)); - acc += 4; - } else if (dispatchId == DispatchId.UINT32) { - fieldValue = - new StaticInvoke( - Integer.class, - "toUnsignedLong", - descriptor.getTypeRef(), - bufferGetInt32(buffer, getBufferIndex(readerIndex, acc))); - acc += 4; - } else if (dispatchId == DispatchId.INT64 || dispatchId == DispatchId.UINT64) { - fieldValue = bufferGetInt64(buffer, getBufferIndex(readerIndex, acc)); - acc += 8; - } else if (dispatchId == DispatchId.FLOAT32) { - fieldValue = bufferGetFloat32(buffer, getBufferIndex(readerIndex, acc)); - acc += 4; - } else if (dispatchId == DispatchId.FLOAT64) { - fieldValue = bufferGetFloat64(buffer, getBufferIndex(readerIndex, acc)); - acc += 8; - } else { - throw new IllegalStateException("Unsupported dispatchId: " + dispatchId); - } - // `bean` will be replaced by `Reference` to cut-off expr dependency. - groupExpressions.add(setFieldValue(bean, descriptor, fieldValue)); - } - if (hasFewFields() || numPrimitiveFields < 4 || isRecord) { - expressions.add(groupExpressions); - } else { - expressions.add( - objectCodecOptimizer.invokeGenerated( - ofHashSet(bean, buffer, readerIndex), groupExpressions, "readFields")); - } - } - Expression increaseReaderIndex = - new Invoke( - buffer, "increaseReaderIndex", new Literal(totalSizeLiteral, PRIMITIVE_INT_TYPE)); - expressions.add(increaseReaderIndex); - return expressions; - } - - private List deserializeCompressedPrimitives( - Expression bean, Expression buffer, List> primitiveGroups) { - List expressions = new ArrayList<>(); - int numPrimitiveFields = getNumPrimitiveFields(primitiveGroups); - for (List group : primitiveGroups) { - // After this check, following read can be totally unsafe without checks. - // checkReadableBytes first, `fillBuffer` may create a new heap buffer. - ReplaceStub checkReadableBytesStub = new ReplaceStub(); - expressions.add(checkReadableBytesStub); - Expression readerIndex = - new Invoke(buffer, "readerIndex", "readerIndex", PRIMITIVE_INT_TYPE); - expressions.add(readerIndex); - ListExpression groupExpressions = new ListExpression(); - int acc = 0; - boolean compressStarted = false; - for (Descriptor descriptor : group) { - int dispatchId = getNumericDescriptorDispatchId(descriptor); - Expression fieldValue; - if (dispatchId == DispatchId.BOOL) { - fieldValue = bufferGetBoolean(buffer, getBufferIndex(readerIndex, acc)); - acc += 1; - } else if (dispatchId == DispatchId.INT8) { - fieldValue = bufferGetByte(buffer, getBufferIndex(readerIndex, acc)); - acc += 1; - } else if (dispatchId == DispatchId.UINT8) { - fieldValue = - new StaticInvoke( - Byte.class, - "toUnsignedInt", - descriptor.getTypeRef(), - bufferGetByte(buffer, getBufferIndex(readerIndex, acc))); - acc += 1; - } else if (dispatchId == DispatchId.CHAR) { - fieldValue = bufferGetChar(buffer, getBufferIndex(readerIndex, acc)); - acc += 2; - } else if (dispatchId == DispatchId.INT16) { - fieldValue = bufferGetInt16(buffer, getBufferIndex(readerIndex, acc)); - acc += 2; - } else if (dispatchId == DispatchId.UINT16) { - fieldValue = - new StaticInvoke( - Short.class, - "toUnsignedInt", - descriptor.getTypeRef(), - bufferGetInt16(buffer, getBufferIndex(readerIndex, acc))); - acc += 2; - } else if (dispatchId == DispatchId.FLOAT16) { - fieldValue = - new StaticInvoke( - Float16.class, - "fromBits", - TypeRef.of(Float16.class), - bufferGetInt16(buffer, getBufferIndex(readerIndex, acc))); - acc += 2; - } else if (dispatchId == DispatchId.BFLOAT16) { - fieldValue = - new StaticInvoke( - BFloat16.class, - "fromBits", - TypeRef.of(BFloat16.class), - bufferGetInt16(buffer, getBufferIndex(readerIndex, acc))); - acc += 2; - } else if (dispatchId == DispatchId.FLOAT32) { - fieldValue = bufferGetFloat32(buffer, getBufferIndex(readerIndex, acc)); - acc += 4; - } else if (dispatchId == DispatchId.FLOAT64) { - fieldValue = bufferGetFloat64(buffer, getBufferIndex(readerIndex, acc)); - acc += 8; - } else if (dispatchId == DispatchId.INT32) { - fieldValue = bufferGetInt32(buffer, getBufferIndex(readerIndex, acc)); - acc += 4; - } else if (dispatchId == DispatchId.UINT32) { - fieldValue = - new StaticInvoke( - Integer.class, - "toUnsignedLong", - descriptor.getTypeRef(), - bufferGetInt32(buffer, getBufferIndex(readerIndex, acc))); - acc += 4; - } else if (dispatchId == DispatchId.INT64 || dispatchId == DispatchId.UINT64) { - fieldValue = bufferGetInt64(buffer, getBufferIndex(readerIndex, acc)); - acc += 8; - } else if (dispatchId == DispatchId.VARINT32) { - if (!compressStarted) { - compressStarted = true; - addIncReaderIndexExpr(groupExpressions, buffer, acc); - } - fieldValue = readVarInt32(buffer); - } else if (dispatchId == DispatchId.VAR_UINT32) { - if (!compressStarted) { - compressStarted = true; - addIncReaderIndexExpr(groupExpressions, buffer, acc); - } - fieldValue = - new StaticInvoke( - Integer.class, - "toUnsignedLong", - descriptor.getTypeRef(), - new Invoke(buffer, "readVarUInt32", PRIMITIVE_INT_TYPE)); - } else if (dispatchId == DispatchId.VARINT64) { - if (!compressStarted) { - compressStarted = true; - addIncReaderIndexExpr(groupExpressions, buffer, acc); - } - fieldValue = new Invoke(buffer, "readVarInt64", PRIMITIVE_LONG_TYPE); - } else if (dispatchId == DispatchId.TAGGED_INT64) { - if (!compressStarted) { - compressStarted = true; - addIncReaderIndexExpr(groupExpressions, buffer, acc); - } - fieldValue = new Invoke(buffer, "readTaggedInt64", PRIMITIVE_LONG_TYPE); - } else if (dispatchId == DispatchId.VAR_UINT64) { - if (!compressStarted) { - compressStarted = true; - addIncReaderIndexExpr(groupExpressions, buffer, acc); - } - fieldValue = new Invoke(buffer, "readVarUInt64", PRIMITIVE_LONG_TYPE); - } else if (dispatchId == DispatchId.TAGGED_UINT64) { - if (!compressStarted) { - compressStarted = true; - addIncReaderIndexExpr(groupExpressions, buffer, acc); - } - fieldValue = new Invoke(buffer, "readTaggedUInt64", PRIMITIVE_LONG_TYPE); - } else { - throw new IllegalStateException("Unsupported dispatchId: " + dispatchId); - } - // `bean` will be replaced by `Reference` to cut-off expr dependency. - groupExpressions.add(setFieldValue(bean, descriptor, fieldValue)); - } - if (acc != 0) { - checkReadableBytesStub.setTargetObject( - new Invoke(buffer, "checkReadableBytes", Literal.ofInt(acc))); - } - if (!compressStarted) { - addIncReaderIndexExpr(groupExpressions, buffer, acc); - } - if (hasFewFields() || numPrimitiveFields < 4 || isRecord) { - expressions.add(groupExpressions); - } else { - expressions.add( - objectCodecOptimizer.invokeGenerated( - ofHashSet(bean, buffer, readerIndex), groupExpressions, "readFields")); - } - } - return expressions; - } - - private void addIncReaderIndexExpr(ListExpression expressions, Expression buffer, int diff) { - if (diff != 0) { - expressions.add(new Invoke(buffer, "increaseReaderIndex", Literal.ofInt(diff))); - } - } - - private Expression getReaderAddress(Expression readerPos, long acc) { - if (acc == 0) { - return readerPos; - } - return add(readerPos, new Literal(acc, PRIMITIVE_LONG_TYPE)); - } - - private Expression getBufferIndex(Expression index, int acc) { - if (acc == 0) { - return index; - } - return add(index, Literal.ofInt(acc)); - } -} diff --git a/java/fory-core/src/main/java25/org/apache/fory/memory/MemoryBuffer.java b/java/fory-core/src/main/java25/org/apache/fory/memory/MemoryBuffer.java index 30c5d7b0e5..5232671e14 100644 --- a/java/fory-core/src/main/java25/org/apache/fory/memory/MemoryBuffer.java +++ b/java/fory-core/src/main/java25/org/apache/fory/memory/MemoryBuffer.java @@ -3844,21 +3844,102 @@ public void copyFrom(int offset, MemoryBuffer source, int sourcePointer, int num source.copyTo(sourcePointer, this, offset, numBytes); } - /** - * JVM-only bulk copy method. Copies {@code numBytes} bytes to target unsafe object and pointer. - * Throws on Android before executing unsafe memory access. - */ - public void copyToUnsafe(long offset, Object target, long targetPointer, int numBytes) { + public void copyToByteArray(int offset, byte[] target, int targetOffset, int numBytes) { if (AndroidSupport.IS_ANDROID) { - MemoryOps.throwRawUnsafeMemoryCopyUnsupported(); + MemoryOps.copyToByteArray(this, offset, target, targetOffset, numBytes); } else { - checkArgument(target != null, "Raw native-address target copy is unsupported on JDK25"); - final long thisPointer = this.address + offset; - checkArgument(thisPointer + numBytes <= addressLimit); - memoryAccess.copyMemory(this.heapMemory, thisPointer, target, targetPointer, numBytes); + checkArrayCopy(offset, targetOffset, target.length, numBytes, 0); + memoryAccess.copyMemory(heapMemory, address + offset, target, targetOffset, numBytes); + } + } + + public void copyToBooleanArray(int offset, boolean[] target, int targetOffset, int numBytes) { + if (AndroidSupport.IS_ANDROID) { + MemoryOps.copyToBooleanArray(this, offset, target, targetOffset, numBytes); + } else { + checkArrayCopy(offset, targetOffset, target.length, numBytes, 0); + memoryAccess.copyMemory(heapMemory, address + offset, target, targetOffset, numBytes); + } + } + + public void copyToCharArray(int offset, char[] target, int targetOffset, int numBytes) { + if (AndroidSupport.IS_ANDROID) { + MemoryOps.copyToCharArray(this, offset, target, targetOffset, numBytes); + } else { + checkArrayCopy(offset, targetOffset, target.length, numBytes, 1); + memoryAccess.copyMemory( + heapMemory, address + offset, target, arrayCopyOffset(targetOffset, 1), numBytes); + } + } + + public void copyToShortArray(int offset, short[] target, int targetOffset, int numBytes) { + if (AndroidSupport.IS_ANDROID) { + MemoryOps.copyToShortArray(this, offset, target, targetOffset, numBytes); + } else { + checkArrayCopy(offset, targetOffset, target.length, numBytes, 1); + memoryAccess.copyMemory( + heapMemory, address + offset, target, arrayCopyOffset(targetOffset, 1), numBytes); + } + } + + public void copyToIntArray(int offset, int[] target, int targetOffset, int numBytes) { + if (AndroidSupport.IS_ANDROID) { + MemoryOps.copyToIntArray(this, offset, target, targetOffset, numBytes); + } else { + checkArrayCopy(offset, targetOffset, target.length, numBytes, 2); + memoryAccess.copyMemory( + heapMemory, address + offset, target, arrayCopyOffset(targetOffset, 2), numBytes); + } + } + + public void copyToLongArray(int offset, long[] target, int targetOffset, int numBytes) { + if (AndroidSupport.IS_ANDROID) { + MemoryOps.copyToLongArray(this, offset, target, targetOffset, numBytes); + } else { + checkArrayCopy(offset, targetOffset, target.length, numBytes, 3); + memoryAccess.copyMemory( + heapMemory, address + offset, target, arrayCopyOffset(targetOffset, 3), numBytes); + } + } + + public void copyToFloatArray(int offset, float[] target, int targetOffset, int numBytes) { + if (AndroidSupport.IS_ANDROID) { + MemoryOps.copyToFloatArray(this, offset, target, targetOffset, numBytes); + } else { + checkArrayCopy(offset, targetOffset, target.length, numBytes, 2); + memoryAccess.copyMemory( + heapMemory, address + offset, target, arrayCopyOffset(targetOffset, 2), numBytes); } } + public void copyToDoubleArray(int offset, double[] target, int targetOffset, int numBytes) { + if (AndroidSupport.IS_ANDROID) { + MemoryOps.copyToDoubleArray(this, offset, target, targetOffset, numBytes); + } else { + checkArrayCopy(offset, targetOffset, target.length, numBytes, 3); + memoryAccess.copyMemory( + heapMemory, address + offset, target, arrayCopyOffset(targetOffset, 3), numBytes); + } + } + + private void checkArrayCopy( + int offset, int targetOffset, int targetLength, int numBytes, int elementShift) { + int elementMask = (1 << elementShift) - 1; + if ((numBytes & elementMask) != 0) { + throw new IllegalArgumentException("numBytes is not aligned to array element size"); + } + int numElements = numBytes >> elementShift; + if ((offset | targetOffset | numBytes | numElements) < 0 + || offset > size - numBytes + || targetOffset > targetLength - numElements) { + throwOOBException(); + } + } + + private static long arrayCopyOffset(int elementOffset, int elementShift) { + return (long) elementOffset << elementShift; + } + /** * JVM-only bulk copy method. Copies {@code numBytes} bytes from source unsafe object and pointer. * Throws on Android before executing unsafe memory access. diff --git a/java/fory-core/src/test/java/org/apache/fory/memory/MemoryBufferTest.java b/java/fory-core/src/test/java/org/apache/fory/memory/MemoryBufferTest.java index 8eced95b1e..f395e60467 100644 --- a/java/fory-core/src/test/java/org/apache/fory/memory/MemoryBufferTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/memory/MemoryBufferTest.java @@ -215,8 +215,9 @@ public static void main(String[] args) { check(source.equalTo(target, 0, 0, 4), true); check(source.getBytes(1, 2), new byte[] {2, 3}); - assertThrows( - UnsupportedOperationException.class, () -> source.copyToUnsafe(0, new byte[4], 0, 4)); + byte[] bytes = new byte[4]; + source.copyToByteArray(0, bytes, 0, 4); + check(bytes, new byte[] {1, 2, 3, 4}); assertThrows( UnsupportedOperationException.class, () -> target.copyFromUnsafe(0, new byte[4], 0, 4)); } @@ -440,6 +441,70 @@ public void testDirectPrimitiveArrays() { assertEquals(readLongs, longs); } + @Test + public void testTypedArrayCopies() { + assertTypedArrayCopies(MemoryUtils.buffer(256)); + assertTypedArrayCopies(MemoryUtils.wrap(ByteBuffer.allocateDirect(256))); + } + + private void assertTypedArrayCopies(MemoryBuffer buffer) { + byte[] bytes = {1, 2, 3, 4}; + int byteOffset = buffer.writerIndex(); + buffer.writeBytes(bytes); + byte[] byteCopy = new byte[bytes.length]; + buffer.copyToByteArray(byteOffset, byteCopy, 0, bytes.length); + assertEquals(byteCopy, bytes); + + boolean[] booleans = {true, false, true}; + int booleanOffset = buffer.writerIndex(); + buffer.writeBooleans(booleans); + boolean[] booleanCopy = new boolean[booleans.length]; + buffer.copyToBooleanArray(booleanOffset, booleanCopy, 0, booleans.length); + assertEquals(booleanCopy, booleans); + + char[] chars = {'a', 0x1234, Character.MAX_VALUE}; + int charOffset = buffer.writerIndex(); + buffer.writeChars(chars); + char[] charCopy = new char[chars.length]; + buffer.copyToCharArray(charOffset, charCopy, 0, chars.length * Character.BYTES); + assertEquals(charCopy, chars); + + short[] shorts = {1, -2, Short.MAX_VALUE}; + int shortOffset = buffer.writerIndex(); + buffer.writeShorts(shorts); + short[] shortCopy = new short[shorts.length]; + buffer.copyToShortArray(shortOffset, shortCopy, 0, shorts.length * Short.BYTES); + assertEquals(shortCopy, shorts); + + int[] ints = {1, -2, Integer.MIN_VALUE}; + int intOffset = buffer.writerIndex(); + buffer.writeInts(ints); + int[] intCopy = new int[ints.length]; + buffer.copyToIntArray(intOffset, intCopy, 0, ints.length * Integer.BYTES); + assertEquals(intCopy, ints); + + long[] longs = {1L, -2L, Long.MAX_VALUE}; + int longOffset = buffer.writerIndex(); + buffer.writeLongs(longs); + long[] longCopy = new long[longs.length]; + buffer.copyToLongArray(longOffset, longCopy, 0, longs.length * Long.BYTES); + assertEquals(longCopy, longs); + + float[] floats = {1.5f, -2.5f, Float.MAX_VALUE}; + int floatOffset = buffer.writerIndex(); + buffer.writeFloats(floats); + float[] floatCopy = new float[floats.length]; + buffer.copyToFloatArray(floatOffset, floatCopy, 0, floats.length * Float.BYTES); + assertEquals(floatCopy, floats); + + double[] doubles = {1.5d, -2.5d, Double.MAX_VALUE}; + int doubleOffset = buffer.writerIndex(); + buffer.writeDoubles(doubles); + double[] doubleCopy = new double[doubles.length]; + buffer.copyToDoubleArray(doubleOffset, doubleCopy, 0, doubles.length * Double.BYTES); + assertEquals(doubleCopy, doubles); + } + @Test public void testWritePrimitiveArrayWithSizeEmbedded() { MemoryBuffer buf = MemoryUtils.buffer(16); diff --git a/java/fory-format/src/main/java/org/apache/fory/format/row/binary/BinaryArray.java b/java/fory-format/src/main/java/org/apache/fory/format/row/binary/BinaryArray.java index fdb85c8f2f..093c61f71f 100644 --- a/java/fory-format/src/main/java/org/apache/fory/format/row/binary/BinaryArray.java +++ b/java/fory-format/src/main/java/org/apache/fory/format/row/binary/BinaryArray.java @@ -187,43 +187,43 @@ public byte[] toBytes() { public boolean[] toBooleanArray() { boolean[] values = new boolean[numElements]; - buffer.copyToUnsafe(elementOffset, values, UnsafeOps.BOOLEAN_ARRAY_OFFSET, numElements); + buffer.copyToBooleanArray(elementOffset, values, 0, numElements); return values; } public byte[] toByteArray() { byte[] values = new byte[numElements]; - buffer.copyToUnsafe(elementOffset, values, UnsafeOps.BYTE_ARRAY_OFFSET, numElements); + buffer.copyToByteArray(elementOffset, values, 0, numElements); return values; } public short[] toShortArray() { short[] values = new short[numElements]; - buffer.copyToUnsafe(elementOffset, values, UnsafeOps.SHORT_ARRAY_OFFSET, numElements * 2); + buffer.copyToShortArray(elementOffset, values, 0, numElements * 2); return values; } public int[] toIntArray() { int[] values = new int[numElements]; - buffer.copyToUnsafe(elementOffset, values, UnsafeOps.INT_ARRAY_OFFSET, numElements * 4); + buffer.copyToIntArray(elementOffset, values, 0, numElements * 4); return values; } public long[] toLongArray() { long[] values = new long[numElements]; - buffer.copyToUnsafe(elementOffset, values, UnsafeOps.LONG_ARRAY_OFFSET, numElements * 8); + buffer.copyToLongArray(elementOffset, values, 0, numElements * 8); return values; } public float[] toFloatArray() { float[] values = new float[numElements]; - buffer.copyToUnsafe(elementOffset, values, UnsafeOps.FLOAT_ARRAY_OFFSET, numElements * 4); + buffer.copyToFloatArray(elementOffset, values, 0, numElements * 4); return values; } public double[] toDoubleArray() { double[] values = new double[numElements]; - buffer.copyToUnsafe(elementOffset, values, UnsafeOps.DOUBLE_ARRAY_OFFSET, numElements * 8); + buffer.copyToDoubleArray(elementOffset, values, 0, numElements * 8); return values; } diff --git a/java/fory-format/src/main/java/org/apache/fory/format/row/binary/BinaryMap.java b/java/fory-format/src/main/java/org/apache/fory/format/row/binary/BinaryMap.java index d607c2fe5b..6469965caf 100644 --- a/java/fory-format/src/main/java/org/apache/fory/format/row/binary/BinaryMap.java +++ b/java/fory-format/src/main/java/org/apache/fory/format/row/binary/BinaryMap.java @@ -25,7 +25,6 @@ import org.apache.fory.format.type.Field; import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.memory.MemoryUtils; -import org.apache.fory.platform.UnsafeOps; /** * An BinaryMap implementation of Map which is backed by two BinaryArray./ForyStructOutput @@ -124,8 +123,8 @@ public MapData copy() { return mapCopy; } - public void writeToMemory(Object target, long targetOffset) { - buf.copyToUnsafe(baseOffset, target, targetOffset, sizeInBytes); + public void writeTo(byte[] target, int targetOffset) { + buf.copyToByteArray(baseOffset, target, targetOffset, sizeInBytes); } public void writeTo(ByteBuffer buffer) { @@ -133,7 +132,7 @@ public void writeTo(ByteBuffer buffer) { byte[] target = buffer.array(); int offset = buffer.arrayOffset(); int pos = buffer.position(); - writeToMemory(target, UnsafeOps.BYTE_ARRAY_OFFSET + offset + pos); + writeTo(target, offset + pos); buffer.position(pos + sizeInBytes); } From 46f632441514cdbe8c0cd900b35776785c515472 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sun, 24 May 2026 23:58:56 +0800 Subject: [PATCH 31/69] feat(java): remove Unsafe for jdk25 --- benchmarks/java/pom.xml | 36 +- .../fory/benchmark/CompressStringSuite.java | 3 +- .../apache/fory/benchmark/Identity2IdMap.java | 15 +- .../fory/benchmark/Jdk25MrJarCheck.java | 12 +- .../apache/fory/benchmark/MemorySuite.java | 12 +- .../fory/benchmark/NewJava11StringSuite.java | 13 +- .../apache/fory/benchmark/NewStringSuite.java | 7 +- .../apache/fory/benchmark/UnsafeAccess.java | 37 + benchmarks/java25/README.md | 8 +- ci/run_ci.sh | 3 +- ci/tasks/java.py | 9 +- docs/guide/java/troubleshooting.md | 45 +- docs/guide/kotlin/configuration.md | 23 + integration_tests/graalvm_tests/pom.xml | 22 +- .../jdk_compatibility_tests/pom.xml | 1 - integration_tests/jpms_tests/pom.xml | 1 - java/fory-core/pom.xml | 18 +- .../fory/builder/BaseObjectCodecBuilder.java | 10 +- .../org/apache/fory/builder/CodecBuilder.java | 83 +- .../fory/builder/ObjectCodecBuilder.java | 5 +- .../org/apache/fory/codegen/Expression.java | 36 +- .../org/apache/fory/memory/LittleEndian.java | 11 +- .../org/apache/fory/memory/MemoryBuffer.java | 330 +++++-- .../org/apache/fory/memory/MemoryOps.java | 86 +- .../org/apache/fory/platform/UnsafeOps.java | 215 ----- .../fory/platform/internal/_JDKAccess.java | 4 + .../apache/fory/reflect/FieldAccessor.java | 87 +- .../apache/fory/reflect/ObjectCreator.java | 2 +- .../apache/fory/reflect/ObjectCreators.java | 23 +- .../apache/fory/reflect/ReflectionUtils.java | 2 +- .../apache/fory/resolver/ClassResolver.java | 16 - .../fory/serializer/CompatibleSerializer.java | 4 +- .../fory/serializer/ExceptionSerializers.java | 2 +- .../serializer/ObjectStreamSerializer.java | 6 +- .../fory/serializer/PlatformStringUtils.java | 30 +- .../apache/fory/serializer/Serializers.java | 101 +- .../org/apache/fory/memory/LittleEndian.java | 65 ++ .../org/apache/fory/memory/MemoryBuffer.java | 902 +++++++----------- .../org/apache/fory/platform/UnsafeOps.java | 497 ---------- .../fory/platform/internal/_JDKAccess.java | 4 + .../src/main/resources/META-INF/LICENSE | 1 - .../fory-core/native-image.properties | 2 +- .../fory/GuavaOptionalDependencyTest.java | 4 +- .../fory/JpmsOptionalClassLoadingTest.java | 14 +- .../test/java/org/apache/fory/StreamTest.java | 9 + .../test/java/org/apache/fory/TestUtils.java | 47 +- .../apache/fory/config/ForyBuilderTest.java | 13 +- .../apache/fory/memory/MemoryBufferTest.java | 69 +- .../resolver/GraalvmRuntimeArrayTest.java | 13 +- .../serializer/AndroidDynamicFeatureTest.java | 10 +- .../org/apache/fory/util/StringUtilsTest.java | 49 +- java/fory-format/pom.xml | 2 +- .../fory/format/row/binary/BinaryArray.java | 118 ++- .../row/binary/writer/BinaryArrayWriter.java | 75 +- .../format/row/binary/BinaryArrayTest.java | 104 ++ .../fory/format/row/binary/BinaryRowTest.java | 13 +- java/fory-simd/pom.xml | 2 +- java/pom.xml | 1 + kotlin/fory-kotlin/pom.xml | 2 + .../serializer/kotlin/KotlinSerializers.java | 18 + .../kotlin/KotlinBuiltinSerializers.kt | 236 +++++ .../kotlin/KotlinDefaultValueSupport.kt | 74 +- .../serializer/kotlin/DefaultValueTest.kt | 7 +- scala/build.sbt | 14 + .../scala/ToFactorySerializers.java | 58 +- 65 files changed, 1986 insertions(+), 1755 deletions(-) create mode 100644 benchmarks/java/src/main/java/org/apache/fory/benchmark/UnsafeAccess.java delete mode 100644 java/fory-core/src/main/java/org/apache/fory/platform/UnsafeOps.java create mode 100644 java/fory-core/src/main/java25/org/apache/fory/memory/LittleEndian.java delete mode 100644 java/fory-core/src/main/java25/org/apache/fory/platform/UnsafeOps.java create mode 100644 kotlin/fory-kotlin/src/main/kotlin/org/apache/fory/serializer/kotlin/KotlinBuiltinSerializers.kt diff --git a/benchmarks/java/pom.xml b/benchmarks/java/pom.xml index 2c8e5f33a1..a2ba15d9cf 100644 --- a/benchmarks/java/pom.xml +++ b/benchmarks/java/pom.xml @@ -264,8 +264,12 @@ src="${project.build.directory}/${uberjar.name}.jar" dest="${jdk25.benchmark.check.dir}"> + + - - + + @@ -297,18 +303,18 @@ - - + if="jdk25.benchmark.rootunsafeops.present" + message="JDK25 benchmark jar must not contain root UnsafeOps class."/> + + @@ -321,12 +327,6 @@ - - diff --git a/benchmarks/java/src/main/java/org/apache/fory/benchmark/CompressStringSuite.java b/benchmarks/java/src/main/java/org/apache/fory/benchmark/CompressStringSuite.java index 0fcdb6b4ed..73054a6438 100644 --- a/benchmarks/java/src/main/java/org/apache/fory/benchmark/CompressStringSuite.java +++ b/benchmarks/java/src/main/java/org/apache/fory/benchmark/CompressStringSuite.java @@ -21,6 +21,7 @@ import java.nio.ByteBuffer; import org.apache.fory.memory.MemoryBuffer; +import org.apache.fory.serializer.StringEncodingUtils; import org.apache.fory.util.StringUtils; import org.openjdk.jmh.Main; import org.openjdk.jmh.annotations.Benchmark; @@ -99,7 +100,7 @@ public Object latinScalarCheck() { @Benchmark public Object latinSuperWordCheck() { - return StringUtils.isLatin(latinStrChars); + return StringEncodingUtils.isLatin(latinStrChars); } public static void main(String[] args) throws Exception { diff --git a/benchmarks/java/src/main/java/org/apache/fory/benchmark/Identity2IdMap.java b/benchmarks/java/src/main/java/org/apache/fory/benchmark/Identity2IdMap.java index 946e98a103..8e5167e031 100644 --- a/benchmarks/java/src/main/java/org/apache/fory/benchmark/Identity2IdMap.java +++ b/benchmarks/java/src/main/java/org/apache/fory/benchmark/Identity2IdMap.java @@ -18,7 +18,6 @@ import java.util.ArrayList; import java.util.List; -import org.apache.fory.platform.UnsafeOps; // Derived from // https://github.com/RuedigerMoeller/fast-serialization/blob/e8da5591daa09452791dcd992ea4f83b20937be7/src/main/java/org/nustaq/serialization/util/FSTIdentity2IdMap.java. @@ -405,20 +404,10 @@ public static void clear(int[] arr, int len) { int count = 0; final int emptyArrayLength = EMPTY_INT_ARRAY.length; while (len - count > emptyArrayLength) { - UnsafeOps.copyMemory( - EMPTY_INT_ARRAY, - UnsafeOps.INT_ARRAY_OFFSET, - arr, - UnsafeOps.INT_ARRAY_OFFSET + count, - emptyArrayLength); + System.arraycopy(EMPTY_INT_ARRAY, 0, arr, count, emptyArrayLength); count += emptyArrayLength; } - UnsafeOps.copyMemory( - EMPTY_INT_ARRAY, - UnsafeOps.INT_ARRAY_OFFSET, - arr, - UnsafeOps.INT_ARRAY_OFFSET + count, - len - count); + System.arraycopy(EMPTY_INT_ARRAY, 0, arr, count, len - count); } public static void clear(Object[] arr, int len) { diff --git a/benchmarks/java/src/main/java/org/apache/fory/benchmark/Jdk25MrJarCheck.java b/benchmarks/java/src/main/java/org/apache/fory/benchmark/Jdk25MrJarCheck.java index 1bb2b38d91..41aae1fdae 100644 --- a/benchmarks/java/src/main/java/org/apache/fory/benchmark/Jdk25MrJarCheck.java +++ b/benchmarks/java/src/main/java/org/apache/fory/benchmark/Jdk25MrJarCheck.java @@ -20,7 +20,6 @@ package org.apache.fory.benchmark; import org.apache.fory.memory.MemoryBuffer; -import org.apache.fory.platform.UnsafeOps; import org.apache.fory.platform.internal._JDKAccess; import org.apache.fory.reflect.FieldAccessor; @@ -30,7 +29,7 @@ private Jdk25MrJarCheck() {} public static void main(String[] args) { verifyClass(MemoryBuffer.class); - verifyClass(UnsafeOps.class); + verifyMissing("org.apache.fory.platform.UnsafeOps"); verifyClass(_JDKAccess.class); verifyClass(FieldAccessor.class); verifyClass("org.apache.fory.serializer.PlatformStringUtils"); @@ -39,6 +38,15 @@ public static void main(String[] args) { } } + private static void verifyMissing(String className) { + try { + Class.forName(className); + throw new IllegalStateException("JDK25 benchmark jar must not contain " + className); + } catch (ClassNotFoundException expected) { + // expected + } + } + private static void verifyClass(String className) { try { verifyClass(Class.forName(className)); diff --git a/benchmarks/java/src/main/java/org/apache/fory/benchmark/MemorySuite.java b/benchmarks/java/src/main/java/org/apache/fory/benchmark/MemorySuite.java index a51441b1f3..f46b5aacc5 100644 --- a/benchmarks/java/src/main/java/org/apache/fory/benchmark/MemorySuite.java +++ b/benchmarks/java/src/main/java/org/apache/fory/benchmark/MemorySuite.java @@ -23,7 +23,6 @@ import java.util.Random; import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.memory.MemoryUtils; -import org.apache.fory.platform.UnsafeOps; import org.apache.fory.util.StringUtils; import org.openjdk.jmh.Main; import org.openjdk.jmh.annotations.BenchmarkMode; @@ -33,10 +32,13 @@ import org.openjdk.jmh.annotations.Scope; import org.openjdk.jmh.annotations.Setup; import org.openjdk.jmh.annotations.State; +import sun.misc.Unsafe; @BenchmarkMode(Mode.Throughput) @CompilerControl(value = CompilerControl.Mode.INLINE) public class MemorySuite { + private static final Unsafe UNSAFE = UnsafeAccess.load(); + private static final int BYTE_ARRAY_OFFSET = UNSAFE.arrayBaseOffset(byte[].class); static int arrLen = 32; static { @@ -138,12 +140,8 @@ public Object systemArrayCopy(MemoryState state) { @org.openjdk.jmh.annotations.Benchmark public Object unsafeCopy(MemoryState state) { - UnsafeOps.UNSAFE.copyMemory( - state.bytes, - UnsafeOps.BYTE_ARRAY_OFFSET, - target, - UnsafeOps.BYTE_ARRAY_OFFSET, - state.bytes.length); + UNSAFE.copyMemory( + state.bytes, BYTE_ARRAY_OFFSET, target, BYTE_ARRAY_OFFSET, state.bytes.length); return target; } diff --git a/benchmarks/java/src/main/java/org/apache/fory/benchmark/NewJava11StringSuite.java b/benchmarks/java/src/main/java/org/apache/fory/benchmark/NewJava11StringSuite.java index c4fcde43e1..762ec646a1 100644 --- a/benchmarks/java/src/main/java/org/apache/fory/benchmark/NewJava11StringSuite.java +++ b/benchmarks/java/src/main/java/org/apache/fory/benchmark/NewJava11StringSuite.java @@ -22,13 +22,14 @@ import org.apache.fory.Fory; import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.platform.JdkVersion; -import org.apache.fory.platform.UnsafeOps; import org.apache.fory.serializer.StringSerializer; import org.apache.fory.util.Preconditions; import org.apache.fory.util.StringUtils; import org.openjdk.jmh.Main; +import sun.misc.Unsafe; public class NewJava11StringSuite { + private static final Unsafe UNSAFE = UnsafeAccess.load(); static String str = StringUtils.random(10); static byte[] strBytes; @@ -36,8 +37,8 @@ public class NewJava11StringSuite { static { if (JdkVersion.MAJOR_VERSION > 8) { - strBytes = (byte[]) UnsafeOps.getObject(str, fieldOffset(String.class, "value")); - coder = UnsafeOps.getByte(str, fieldOffset(String.class, "coder")); + strBytes = (byte[]) UNSAFE.getObject(str, fieldOffset(String.class, "value")); + coder = UNSAFE.getByte(str, fieldOffset(String.class, "coder")); } } @@ -56,7 +57,7 @@ public class NewJava11StringSuite { private static long fieldOffset(Class type, String fieldName) { try { - return UnsafeOps.objectFieldOffset(type.getDeclaredField(fieldName)); + return UNSAFE.objectFieldOffset(type.getDeclaredField(fieldName)); } catch (NoSuchFieldException e) { throw new IllegalStateException(e); } @@ -70,8 +71,8 @@ public Object createJDK11StringByCopyStr() { // @Benchmark public Object createJDK11StringByUnsafe() { String str = new String(stubStr); - UnsafeOps.putObject(str, STRING_VALUE_FIELD_OFFSET, strBytes); - UnsafeOps.putObject(str, STRING_CODER_FIELD_OFFSET, coder); + UNSAFE.putObject(str, STRING_VALUE_FIELD_OFFSET, strBytes); + UNSAFE.putByte(str, STRING_CODER_FIELD_OFFSET, coder); return str; } diff --git a/benchmarks/java/src/main/java/org/apache/fory/benchmark/NewStringSuite.java b/benchmarks/java/src/main/java/org/apache/fory/benchmark/NewStringSuite.java index dfbad5136a..43e9fe4a1f 100644 --- a/benchmarks/java/src/main/java/org/apache/fory/benchmark/NewStringSuite.java +++ b/benchmarks/java/src/main/java/org/apache/fory/benchmark/NewStringSuite.java @@ -19,12 +19,13 @@ package org.apache.fory.benchmark; -import org.apache.fory.platform.UnsafeOps; import org.apache.fory.serializer.StringSerializer; import org.apache.fory.util.StringUtils; import org.openjdk.jmh.Main; +import sun.misc.Unsafe; public class NewStringSuite { + private static final Unsafe UNSAFE = UnsafeAccess.load(); static String str = StringUtils.random(230); static char[] strData = str.toCharArray(); @@ -45,7 +46,7 @@ public Object createJDK8StringByCopy() { private static long fieldOffset(Class type, String fieldName) { try { - return UnsafeOps.objectFieldOffset(type.getDeclaredField(fieldName)); + return UNSAFE.objectFieldOffset(type.getDeclaredField(fieldName)); } catch (NoSuchFieldException e) { throw new IllegalStateException(e); } @@ -54,7 +55,7 @@ private static long fieldOffset(Class type, String fieldName) { // @Benchmark public Object createJDK8StringByUnsafe() { String str = new String(stubStr); - UnsafeOps.putObject(str, STRING_VALUE_FIELD_OFFSET, strData); + UNSAFE.putObject(str, STRING_VALUE_FIELD_OFFSET, strData); return str; } diff --git a/benchmarks/java/src/main/java/org/apache/fory/benchmark/UnsafeAccess.java b/benchmarks/java/src/main/java/org/apache/fory/benchmark/UnsafeAccess.java new file mode 100644 index 0000000000..c99cc81183 --- /dev/null +++ b/benchmarks/java/src/main/java/org/apache/fory/benchmark/UnsafeAccess.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.benchmark; + +import java.lang.reflect.Field; +import sun.misc.Unsafe; + +final class UnsafeAccess { + private UnsafeAccess() {} + + static Unsafe load() { + try { + Field field = Unsafe.class.getDeclaredField("theUnsafe"); + field.setAccessible(true); + return (Unsafe) field.get(null); + } catch (ReflectiveOperationException e) { + throw new ExceptionInInitializerError(e); + } + } +} diff --git a/benchmarks/java25/README.md b/benchmarks/java25/README.md index a7edad7c00..80e2928f42 100644 --- a/benchmarks/java25/README.md +++ b/benchmarks/java25/README.md @@ -1,6 +1,6 @@ # Java 25 Direct Memory Access Benchmark -This temporary JMH module compares direct-buffer scalar access paths used to reason about +This diagnostic JMH module compares direct-buffer scalar access paths used to reason about `MemoryBuffer` on JDK 25: - `MemorySegment.get/set` with native-order unaligned layouts. @@ -29,12 +29,16 @@ java -jar target/java25-memory-access-benchmarks.jar \ -f 1 -wi 5 -i 5 -t 1 -w 1s -r 1s ``` -The benchmark class adds the required fork JVM options for the Unsafe path: +The benchmark class adds fork JVM options only for the explicit Unsafe baseline: ```text --add-opens=java.base/java.nio=ALL-UNNAMED --sun-misc-unsafe-memory-access=allow ``` +These flags are not part of Fory's JDK25 zero-Unsafe deployment contract. They are present here so +the benchmark can compare supported `ByteBuffer`/FFM access with the old raw-address Unsafe +baseline in the same process shape. + If you run with `-f 0`, pass those options to the outer `java` command because JMH will not fork a child JVM. diff --git a/ci/run_ci.sh b/ci/run_ci.sh index dfe1611726..76f9aecbd3 100755 --- a/ci/run_ci.sh +++ b/ci/run_ci.sh @@ -113,7 +113,6 @@ jdk25_deny_options() { printf " %s" "--add-opens=java.base/java.util.concurrent.atomic=${fory_open_targets}" printf " %s" "--add-opens=java.base/java.io=${fory_open_targets}" printf " %s" "--add-opens=java.base/java.net=${fory_open_targets}" - printf " %s" "--add-opens=java.base/java.nio=${fory_open_targets}" printf " %s" "--add-opens=java.base/java.math=${fory_open_targets}" } @@ -205,7 +204,7 @@ jdk17_plus_tests() { else java_major=$(echo "$java_version" | cut -d. -f1) fi - JDK_JAVA_OPTIONS="--add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED" + JDK_JAVA_OPTIONS="--add-opens=java.base/java.nio=org.apache.arrow.memory.core" if [[ "$java_major" -ge 25 ]]; then JDK_JAVA_OPTIONS="$JDK_JAVA_OPTIONS $(jdk25_deny_options) $(jdk25_javac_options)" fi diff --git a/ci/tasks/java.py b/ci/tasks/java.py index bed7db53fc..2aca7ad071 100644 --- a/ci/tasks/java.py +++ b/ci/tasks/java.py @@ -89,7 +89,6 @@ def jdk25_deny_options(): f"--add-opens=java.base/java.util.concurrent.atomic={fory_open_targets}", f"--add-opens=java.base/java.io={fory_open_targets}", f"--add-opens=java.base/java.net={fory_open_targets}", - f"--add-opens=java.base/java.nio={fory_open_targets}", f"--add-opens=java.base/java.math={fory_open_targets}", ] @@ -252,12 +251,14 @@ def run_jdk17_plus(java_version="17"): """Run Java 17+ tests.""" logging.info(f"Executing fory java tests with Java {java_version}") common.exec_cmd("java -version") - jdk_options = [ - "--add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED" - ] + jdk_options = [] if java_version == "25": jdk_options.extend(jdk25_deny_options()) jdk_options.extend(jdk25_javac_options()) + else: + jdk_options.append( + "--add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED" + ) os.environ["JDK_JAVA_OPTIONS"] = " ".join(jdk_options) common.cd_project_subdir("java") diff --git a/docs/guide/java/troubleshooting.md b/docs/guide/java/troubleshooting.md index a28518ffa7..9c35b3eb70 100644 --- a/docs/guide/java/troubleshooting.md +++ b/docs/guide/java/troubleshooting.md @@ -150,7 +150,8 @@ fory.registerSerializer(MyClass.class, new MyClassSerializer(fory.getTypeResolve ### JDK25+ zero-Unsafe mode and module opens -When running on JDK25+ with Unsafe memory access denied, start the JVM with: +When running on JDK25+ with Unsafe memory access denied, or on a later JDK where denied Unsafe +memory access becomes the default, start the JVM with: ```bash --sun-misc-unsafe-memory-access=deny @@ -163,31 +164,35 @@ When any Fory artifact is on the classpath instead of the module path, also incl --add-opens=/=ALL-UNNAMED,org.apache.fory.core,org.apache.fory.format ``` -Some optimized serializers and direct-buffer helpers also need JDK-private packages. Add only the -opens needed by the paths used in your process: - -| Path | Required opens | -| -------------------------------------------------------------------------------- | --------------------------------------------------------------- | -| String fast paths and throwable fields | `java.base/java.lang` | -| Serialized lambdas | `java.base/java.lang.invoke` | -| Reflection-based object construction | `java.base/java.lang.reflect`, `java.base/jdk.internal.reflect` | -| Collection wrappers, sublists, `EnumMap`, and `StringTokenizer` | `java.base/java.util` | -| Blocking queue capacity serializers | `java.base/java.util.concurrent` | -| `ByteArrayInputStream`, `ByteArrayOutputStream`, and Java object-stream metadata | `java.base/java.io` | -| Proxy serializers | `java.base/java.lang.reflect` | -| Direct `ByteBuffer` wrapping | `java.base/java.nio` | - -For example, direct `ByteBuffer` wrapping on the module path requires: +Some optimized serializers also need JDK-private packages. For each package in the table, open the +owning JDK module/package to `org.apache.fory.core` and `org.apache.fory.format`; include +`ALL-UNNAMED` too when any Fory artifact is on the classpath. Add only the opens needed by the paths +used in your process: + +| Path | Required opens | +| -------------------------------------------------------------------------------- | ------------------------------------------------------------------------- | +| String fast paths and throwable fields | `java.base/java.lang` | +| Serialized lambdas | `java.base/java.lang.invoke` | +| Reflection-based object construction | `java.base/java.lang.reflect`, `java.base/jdk.internal.reflect` | +| Collection wrappers, sublists, `EnumMap`, and `StringTokenizer` | `java.base/java.util` | +| Blocking queue capacity serializers | `java.base/java.util.concurrent`, `java.base/java.util.concurrent.atomic` | +| `ByteArrayInputStream`, `ByteArrayOutputStream`, and Java object-stream metadata | `java.base/java.io` | +| URL and networking serializers | `java.base/java.net` | +| Proxy serializers | `java.base/java.lang.reflect` | +| Big number internals | `java.base/java.math` | + +Normal classes with final instance fields require final-field mutation to be enabled for the module +that contains Fory's mutating code when Unsafe allocation is denied. Use the Fory module name on the +module path: ```bash ---add-opens=java.base/java.nio=ALL-UNNAMED,org.apache.fory.core,org.apache.fory.format +--enable-final-field-mutation=org.apache.fory.core ``` -Normal classes with final instance fields require final-field mutation to be enabled for Fory core -when Unsafe allocation is denied: +Use `ALL-UNNAMED` when running Fory on the classpath: ```bash ---enable-final-field-mutation=org.apache.fory.core +--enable-final-field-mutation=ALL-UNNAMED ``` Fory restores those final fields through method-handle based access. Non-final fields can still be diff --git a/docs/guide/kotlin/configuration.md b/docs/guide/kotlin/configuration.md index fc44d872ea..8f338e529e 100644 --- a/docs/guide/kotlin/configuration.md +++ b/docs/guide/kotlin/configuration.md @@ -97,6 +97,29 @@ val fory: ThreadSafeFory = ForyKotlin.builder() All configuration options from Fory Java are available. See [Java Configuration](../java/configuration.md) for the complete list. +## JDK25+ Zero-Unsafe Mode + +On JDK25+ with Unsafe memory access denied, Kotlin classes with final constructor properties need +bindable constructor metadata so Fory can call the primary constructor instead of allocating an +uninitialized instance. Enable Java parameter metadata for Kotlin compilation: + +```kotlin +kotlin { + compilerOptions { + javaParameters = true + } +} +``` + +For Maven builds, configure the Kotlin Maven plugin with: + +```xml +true +``` + +The JVM also needs the module opens and final-field mutation option listed in +[Java Troubleshooting](../java/troubleshooting.md#jdk25-zero-unsafe-mode-and-module-opens). + Common options for Kotlin native-mode payloads: ```kotlin diff --git a/integration_tests/graalvm_tests/pom.xml b/integration_tests/graalvm_tests/pom.xml index a0de283a90..906347fdfb 100644 --- a/integration_tests/graalvm_tests/pom.xml +++ b/integration_tests/graalvm_tests/pom.xml @@ -185,7 +185,6 @@ -J--add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED -J--add-opens=java.base/java.io=ALL-UNNAMED -J--add-opens=java.base/java.net=ALL-UNNAMED - -J--add-opens=java.base/java.nio=ALL-UNNAMED -J--add-opens=java.base/java.math=ALL-UNNAMED @@ -198,13 +197,26 @@ java-agent - java + exec test - ${mainClass} - true - false + ${java.home}/bin/java + + --add-opens=java.base/java.lang=ALL-UNNAMED + --add-opens=java.base/java.lang.invoke=ALL-UNNAMED + --add-opens=java.base/java.lang.reflect=ALL-UNNAMED + --add-opens=java.base/jdk.internal.reflect=ALL-UNNAMED + --add-opens=java.base/java.util=ALL-UNNAMED + --add-opens=java.base/java.util.concurrent=ALL-UNNAMED + --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED + --add-opens=java.base/java.io=ALL-UNNAMED + --add-opens=java.base/java.net=ALL-UNNAMED + --add-opens=java.base/java.math=ALL-UNNAMED + -classpath + + ${mainClass} + diff --git a/integration_tests/jdk_compatibility_tests/pom.xml b/integration_tests/jdk_compatibility_tests/pom.xml index 1d0899d92f..53a40d1b56 100644 --- a/integration_tests/jdk_compatibility_tests/pom.xml +++ b/integration_tests/jdk_compatibility_tests/pom.xml @@ -104,7 +104,6 @@ --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED - --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.math=ALL-UNNAMED diff --git a/integration_tests/jpms_tests/pom.xml b/integration_tests/jpms_tests/pom.xml index e98b2d633c..f74ff38671 100644 --- a/integration_tests/jpms_tests/pom.xml +++ b/integration_tests/jpms_tests/pom.xml @@ -99,7 +99,6 @@ --add-opens=java.base/java.util.concurrent.atomic=org.apache.fory.core,org.apache.fory.format --add-opens=java.base/java.io=org.apache.fory.core,org.apache.fory.format --add-opens=java.base/java.net=org.apache.fory.core,org.apache.fory.format - --add-opens=java.base/java.nio=org.apache.fory.core,org.apache.fory.format --add-opens=java.base/java.math=org.apache.fory.core,org.apache.fory.format diff --git a/java/fory-core/pom.xml b/java/fory-core/pom.xml index 6e74e2c2a4..7aabd48d97 100644 --- a/java/fory-core/pom.xml +++ b/java/fory-core/pom.xml @@ -239,7 +239,9 @@ src="${project.build.directory}/${project.build.finalName}.jar" dest="${jdk25.mrjar.check.dir}"> + + @@ -251,9 +253,15 @@ + + @@ -276,8 +284,14 @@ file="${jdk25.mrjar.check.dir}/META-INF/versions/25/org/apache/fory/serializer/PlatformStringUtils.class" property="jdk25.platformstring.present"/> + if="jdk25.root.unsafeops.present" + message="Root UnsafeOps class must not be packaged in fory-core."/> + + diff --git a/java/fory-core/src/main/java/org/apache/fory/builder/BaseObjectCodecBuilder.java b/java/fory-core/src/main/java/org/apache/fory/builder/BaseObjectCodecBuilder.java index 0848c5d2db..bdd81edaf9 100644 --- a/java/fory-core/src/main/java/org/apache/fory/builder/BaseObjectCodecBuilder.java +++ b/java/fory-core/src/main/java/org/apache/fory/builder/BaseObjectCodecBuilder.java @@ -121,7 +121,7 @@ import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.meta.TypeExtMeta; import org.apache.fory.platform.GraalvmSupport; -import org.apache.fory.platform.UnsafeOps; +import org.apache.fory.platform.internal._JDKAccess; import org.apache.fory.reflect.ReflectionUtils; import org.apache.fory.reflect.TypeRef; import org.apache.fory.resolver.ClassResolver; @@ -153,6 +153,7 @@ import org.apache.fory.type.Types; import org.apache.fory.util.Preconditions; import org.apache.fory.util.StringUtils; +import sun.misc.Unsafe; /** * Generate sequential read/write code for java serialization to speed up performance. It also @@ -420,7 +421,12 @@ protected void registerJITNotifyCallback() { */ protected void addCommonImports() { ctx.addImports( - Fory.class, MemoryBuffer.class, WriteContext.class, ReadContext.class, UnsafeOps.class); + Fory.class, + MemoryBuffer.class, + WriteContext.class, + ReadContext.class, + _JDKAccess.class, + Unsafe.class); ctx.addImports(TypeInfo.class, TypeInfoHolder.class, ClassResolver.class); ctx.addImport(Generated.class); ctx.addImports(LazyInitBeanSerializer.class, EnumSerializer.class); diff --git a/java/fory-core/src/main/java/org/apache/fory/builder/CodecBuilder.java b/java/fory-core/src/main/java/org/apache/fory/builder/CodecBuilder.java index 3bcad7a701..dca762854b 100644 --- a/java/fory-core/src/main/java/org/apache/fory/builder/CodecBuilder.java +++ b/java/fory-core/src/main/java/org/apache/fory/builder/CodecBuilder.java @@ -57,7 +57,7 @@ import org.apache.fory.memory.NativeByteOrder; import org.apache.fory.platform.GraalvmSupport; import org.apache.fory.platform.JdkVersion; -import org.apache.fory.platform.UnsafeOps; +import org.apache.fory.platform.internal._JDKAccess; import org.apache.fory.reflect.FieldAccessor; import org.apache.fory.reflect.ObjectCreator; import org.apache.fory.reflect.ObjectCreators; @@ -71,6 +71,7 @@ import org.apache.fory.util.function.Functions; import org.apache.fory.util.record.RecordComponent; import org.apache.fory.util.record.RecordUtils; +import sun.misc.Unsafe; /** * Base builder for generating code to serialize java bean in row-format or object stream format. @@ -199,8 +200,8 @@ protected Reference getRecordCtrHandle() { } protected Expression buildDefaultComponentsArray() { - return new StaticInvoke( - UnsafeOps.class, "copyObjectArray", OBJECT_ARRAY_TYPE, recordComponentDefaultValues); + return new Cast( + new Invoke(recordComponentDefaultValues, "clone", OBJECT_TYPE), OBJECT_ARRAY_TYPE); } /** Returns an expression that get field value from bean. */ @@ -328,22 +329,16 @@ private Expression unsafeAccessField( Expression fieldOffsetExpr = fieldOffsetExpr(cls, descriptor); boolean fieldNullable = fieldNullable(descriptor); if (descriptor.getTypeRef().isPrimitive()) { - // ex: UnsafeOps.getFloat(obj, fieldOffset) + // ex: Unsafe.getFloat(obj, fieldOffset) Preconditions.checkArgument(!fieldNullable); TypeRef returnType = descriptor.getTypeRef(); String funcName = "get" + StringUtils.capitalize(descriptor.getRawType().toString()); - return new StaticInvoke( - UnsafeOps.class, funcName, returnType, false, inputObject, fieldOffsetExpr); + return new Invoke(getUnsafe(), funcName, returnType, false, inputObject, fieldOffsetExpr); } else { - // ex: UnsafeOps.getObject(obj, fieldOffset) - StaticInvoke getObj = - new StaticInvoke( - UnsafeOps.class, - "getObject", - OBJECT_TYPE, - fieldNullable, - inputObject, - fieldOffsetExpr); + // ex: Unsafe.getObject(obj, fieldOffset) + Invoke getObj = + new Invoke( + getUnsafe(), "getObject", OBJECT_TYPE, fieldNullable, inputObject, fieldOffsetExpr); return tryCastIfPublic(getObj, descriptor.getTypeRef(), fieldName); } } @@ -361,15 +356,23 @@ private Expression fieldOffsetExpr(Class cls, Descriptor descriptor) { Expression classExpr = beanClassExpr(field.getDeclaringClass()); new Invoke(classExpr, "getDeclaredField", TypeRef.of(Field.class)); Expression reflectFieldRef = getReflectField(field.getDeclaringClass(), field, false); - return new StaticInvoke( - UnsafeOps.class, "objectFieldOffset", PRIMITIVE_LONG_TYPE, reflectFieldRef) + return new Invoke( + getUnsafe(), "objectFieldOffset", PRIMITIVE_LONG_TYPE, reflectFieldRef) .inline(); }); } else { - return Literal.ofLong(UnsafeOps.objectFieldOffset(field)); + return Literal.ofLong(_JDKAccess.UNSAFE.objectFieldOffset(field)); } } + private Reference getUnsafe() { + return getOrCreateField( + true, + Unsafe.class, + "_unsafe_", + () -> new StaticInvoke(_JDKAccess.class, "unsafe", TypeRef.of(Unsafe.class))); + } + private Reference getFieldAccessor(Descriptor descriptor) { Field field = descriptor.getField(); String fieldName = descriptor.getName(); @@ -474,9 +477,9 @@ private Expression unsafeSetField(Expression bean, Descriptor descriptor, Expres if (descriptor.getTypeRef().isPrimitive()) { Preconditions.checkArgument(getRawType(value.type()) == getRawType(fieldType)); String funcName = "put" + StringUtils.capitalize(getRawType(fieldType).toString()); - return new StaticInvoke(UnsafeOps.class, funcName, bean, fieldOffsetExpr, value); + return new Invoke(getUnsafe(), funcName, bean, fieldOffsetExpr, value); } else { - return new StaticInvoke(UnsafeOps.class, "putObject", bean, fieldOffsetExpr, value); + return new Invoke(getUnsafe(), "putObject", bean, fieldOffsetExpr, value); } } @@ -550,7 +553,9 @@ protected Expression newBean() { Invoke newInstance = new Invoke(getObjectCreator(beanClass), "newInstance", OBJECT_TYPE); return sourcePublicAccessible(beanClass) ? new Cast(newInstance, beanType) : newInstance; } - return new StaticInvoke(UnsafeOps.class, "newInstance", OBJECT_TYPE, beanClassExpr()); + Invoke newInstance = + new Invoke(getUnsafe(), "allocateInstance", OBJECT_TYPE, beanClassExpr()); + return sourcePublicAccessible(beanClass) ? new Cast(newInstance, beanType) : newInstance; } } @@ -649,50 +654,49 @@ private StaticInvoke inlineReflectionUtilsInvoke( /** Build unsafePut operation. */ protected Expression unsafePut(Expression base, Expression pos, Expression value) { - return new StaticInvoke(UnsafeOps.class, "putByte", base, pos, value); + return new Invoke(getUnsafe(), "putByte", base, pos, value); } protected Expression unsafePutBoolean(Expression base, Expression pos, Expression value) { - return new StaticInvoke(UnsafeOps.class, "putBoolean", base, pos, value); + return new Invoke(getUnsafe(), "putBoolean", base, pos, value); } protected Expression unsafePutChar(Expression base, Expression pos, Expression value) { - return new StaticInvoke(UnsafeOps.class, "putChar", base, pos, value); + return new Invoke(getUnsafe(), "putChar", base, pos, value); } protected Expression unsafePutShort(Expression base, Expression pos, Expression value) { - return new StaticInvoke(UnsafeOps.class, "putShort", base, pos, value); + return new Invoke(getUnsafe(), "putShort", base, pos, value); } protected Expression unsafePutInt(Expression base, Expression pos, Expression value) { - return new StaticInvoke(UnsafeOps.class, "putInt", base, pos, value); + return new Invoke(getUnsafe(), "putInt", base, pos, value); } protected Expression unsafePutLong(Expression base, Expression pos, Expression value) { - return new StaticInvoke(UnsafeOps.class, "putLong", base, pos, value); + return new Invoke(getUnsafe(), "putLong", base, pos, value); } protected Expression unsafePutFloat(Expression base, Expression pos, Expression value) { - return new StaticInvoke(UnsafeOps.class, "putFloat", base, pos, value); + return new Invoke(getUnsafe(), "putFloat", base, pos, value); } /** Build unsafePutDouble operation. */ protected Expression unsafePutDouble(Expression base, Expression pos, Expression value) { - return new StaticInvoke(UnsafeOps.class, "putDouble", base, pos, value); + return new Invoke(getUnsafe(), "putDouble", base, pos, value); } /** Build unsafeGet operation. */ protected Expression unsafeGet(Expression base, Expression pos) { - return new StaticInvoke(UnsafeOps.class, "getByte", PRIMITIVE_BYTE_TYPE, base, pos); + return new Invoke(getUnsafe(), "getByte", PRIMITIVE_BYTE_TYPE, base, pos); } protected Expression unsafeGetBoolean(Expression base, Expression pos) { - return new StaticInvoke(UnsafeOps.class, "getBoolean", PRIMITIVE_BOOLEAN_TYPE, base, pos); + return new Invoke(getUnsafe(), "getBoolean", PRIMITIVE_BOOLEAN_TYPE, base, pos); } protected Expression unsafeGetChar(Expression base, Expression pos) { - StaticInvoke expr = - new StaticInvoke(UnsafeOps.class, "getChar", PRIMITIVE_CHAR_TYPE, base, pos); + Inlineable expr = new Invoke(getUnsafe(), "getChar", PRIMITIVE_CHAR_TYPE, base, pos); if (!NativeByteOrder.IS_LITTLE_ENDIAN) { expr = new StaticInvoke(Character.class, "reverseBytes", PRIMITIVE_CHAR_TYPE, expr.inline()); } @@ -700,8 +704,7 @@ protected Expression unsafeGetChar(Expression base, Expression pos) { } protected Expression unsafeGetShort(Expression base, Expression pos) { - StaticInvoke expr = - new StaticInvoke(UnsafeOps.class, "getShort", PRIMITIVE_SHORT_TYPE, base, pos); + Inlineable expr = new Invoke(getUnsafe(), "getShort", PRIMITIVE_SHORT_TYPE, base, pos); if (!NativeByteOrder.IS_LITTLE_ENDIAN) { expr = new StaticInvoke(Short.class, "reverseBytes", PRIMITIVE_SHORT_TYPE, expr.inline()); } @@ -709,7 +712,7 @@ protected Expression unsafeGetShort(Expression base, Expression pos) { } protected Expression unsafeGetInt(Expression base, Expression pos) { - StaticInvoke expr = new StaticInvoke(UnsafeOps.class, "getInt", PRIMITIVE_INT_TYPE, base, pos); + Inlineable expr = new Invoke(getUnsafe(), "getInt", PRIMITIVE_INT_TYPE, base, pos); if (!NativeByteOrder.IS_LITTLE_ENDIAN) { expr = new StaticInvoke(Integer.class, "reverseBytes", PRIMITIVE_INT_TYPE, expr.inline()); } @@ -717,8 +720,7 @@ protected Expression unsafeGetInt(Expression base, Expression pos) { } protected Expression unsafeGetLong(Expression base, Expression pos) { - StaticInvoke expr = - new StaticInvoke(UnsafeOps.class, "getLong", PRIMITIVE_LONG_TYPE, base, pos); + Inlineable expr = new Invoke(getUnsafe(), "getLong", PRIMITIVE_LONG_TYPE, base, pos); if (!NativeByteOrder.IS_LITTLE_ENDIAN) { expr = new StaticInvoke(Long.class, "reverseBytes", PRIMITIVE_LONG_TYPE, expr.inline()); } @@ -726,7 +728,7 @@ protected Expression unsafeGetLong(Expression base, Expression pos) { } protected Expression unsafeGetFloat(Expression base, Expression pos) { - StaticInvoke expr = new StaticInvoke(UnsafeOps.class, "getInt", PRIMITIVE_INT_TYPE, base, pos); + Inlineable expr = new Invoke(getUnsafe(), "getInt", PRIMITIVE_INT_TYPE, base, pos); if (!NativeByteOrder.IS_LITTLE_ENDIAN) { expr = new StaticInvoke(Integer.class, "reverseBytes", PRIMITIVE_INT_TYPE, expr.inline()); } @@ -734,8 +736,7 @@ protected Expression unsafeGetFloat(Expression base, Expression pos) { } protected Expression unsafeGetDouble(Expression base, Expression pos) { - StaticInvoke expr = - new StaticInvoke(UnsafeOps.class, "getLong", PRIMITIVE_LONG_TYPE, base, pos); + Inlineable expr = new Invoke(getUnsafe(), "getLong", PRIMITIVE_LONG_TYPE, base, pos); if (!NativeByteOrder.IS_LITTLE_ENDIAN) { expr = new StaticInvoke(Long.class, "reverseBytes", PRIMITIVE_LONG_TYPE, expr.inline()); } diff --git a/java/fory-core/src/main/java/org/apache/fory/builder/ObjectCodecBuilder.java b/java/fory-core/src/main/java/org/apache/fory/builder/ObjectCodecBuilder.java index f23801f948..b1d25fcbff 100644 --- a/java/fory-core/src/main/java/org/apache/fory/builder/ObjectCodecBuilder.java +++ b/java/fory-core/src/main/java/org/apache/fory/builder/ObjectCodecBuilder.java @@ -65,7 +65,6 @@ import org.apache.fory.logging.LoggerFactory; import org.apache.fory.meta.TypeDef; import org.apache.fory.platform.JdkVersion; -import org.apache.fory.platform.UnsafeOps; import org.apache.fory.reflect.ObjectCreator; import org.apache.fory.reflect.ObjectCreators; import org.apache.fory.reflect.TypeRef; @@ -1187,8 +1186,8 @@ protected void deserializeReadGroup( } protected Expression buildComponentsArray() { - return new StaticInvoke( - UnsafeOps.class, "copyObjectArray", OBJECT_ARRAY_TYPE, recordComponentDefaultValues); + return new Cast( + new Invoke(recordComponentDefaultValues, "clone", OBJECT_TYPE), OBJECT_ARRAY_TYPE); } protected Expression createRecord(SortedMap recordComponents) { diff --git a/java/fory-core/src/main/java/org/apache/fory/codegen/Expression.java b/java/fory-core/src/main/java/org/apache/fory/codegen/Expression.java index d7cdfcb91a..db3439023a 100644 --- a/java/fory-core/src/main/java/org/apache/fory/codegen/Expression.java +++ b/java/fory-core/src/main/java/org/apache/fory/codegen/Expression.java @@ -59,7 +59,10 @@ import java.util.Locale; import java.util.stream.Collectors; import java.util.stream.Stream; -import org.apache.fory.platform.UnsafeOps; +import org.apache.fory.platform.JdkVersion; +import org.apache.fory.platform.internal._JDKAccess; +import org.apache.fory.reflect.ObjectCreator; +import org.apache.fory.reflect.ObjectCreators; import org.apache.fory.reflect.ReflectionUtils; import org.apache.fory.reflect.TypeRef; import org.apache.fory.type.TypeUtils; @@ -1498,14 +1501,31 @@ public ExprCode doGenCode(CodegenContext ctx) { if (arguments.isEmpty() && !ReflectionUtils.hasPublicNoArgConstructor(rawType)) { // janino doesn't generics, so we cast manually. String instance = ctx.newName("instance"); + String target; + String functionName; + String args; + if (JdkVersion.MAJOR_VERSION >= 25) { + String creator = ctx.newName("objectCreator"); + codeBuilder + .append( + ExpressionUtils.callFunc( + ctx.type(ObjectCreator.class), + creator, + ctx.type(ObjectCreators.class), + "getObjectCreator", + clzName + ".class", + false)) + .append('\n'); + target = creator; + functionName = "newInstance"; + args = ""; + } else { + target = ctx.type(_JDKAccess.class) + ".unsafe()"; + functionName = "allocateInstance"; + args = clzName + ".class"; + } String code = - ExpressionUtils.callFunc( - "Object", - instance, - ctx.type(UnsafeOps.class), - "newInstance", - clzName + ".class", - false); + ExpressionUtils.callFunc("Object", instance, target, functionName, args, true); codeBuilder.append(code).append('\n'); String cast = StringUtils.format( diff --git a/java/fory-core/src/main/java/org/apache/fory/memory/LittleEndian.java b/java/fory-core/src/main/java/org/apache/fory/memory/LittleEndian.java index 0e0c2d246a..1b58ed70d6 100644 --- a/java/fory-core/src/main/java/org/apache/fory/memory/LittleEndian.java +++ b/java/fory-core/src/main/java/org/apache/fory/memory/LittleEndian.java @@ -1,7 +1,8 @@ package org.apache.fory.memory; import org.apache.fory.platform.AndroidSupport; -import org.apache.fory.platform.UnsafeOps; +import org.apache.fory.platform.internal._JDKAccess; +import sun.misc.Unsafe; /* * Licensed to the Apache Software Foundation (ASF) under one @@ -23,6 +24,10 @@ */ public class LittleEndian { + private static final Unsafe UNSAFE = AndroidSupport.IS_ANDROID ? null : _JDKAccess.UNSAFE; + private static final int BYTE_ARRAY_OFFSET = + AndroidSupport.IS_ANDROID ? 0 : UNSAFE.arrayBaseOffset(byte[].class); + public static int putVarUint36Small(byte[] arr, int index, long v) { if (v >>> 7 == 0) { arr[index] = (byte) v; @@ -62,7 +67,7 @@ public static long getInt64(byte[] o, int index) { if (AndroidSupport.IS_ANDROID) { return MemoryOps.getInt64(o, index); } - long v = UnsafeOps.getLong(o, UnsafeOps.BYTE_ARRAY_OFFSET + index); + long v = UNSAFE.getLong(o, BYTE_ARRAY_OFFSET + index); return NativeByteOrder.IS_LITTLE_ENDIAN ? v : Long.reverseBytes(v); } @@ -74,6 +79,6 @@ public static void putInt64(byte[] o, int index, long value) { if (!NativeByteOrder.IS_LITTLE_ENDIAN) { value = Long.reverseBytes(value); } - UnsafeOps.putLong(o, UnsafeOps.BYTE_ARRAY_OFFSET + index, value); + UNSAFE.putLong(o, BYTE_ARRAY_OFFSET + index, value); } } diff --git a/java/fory-core/src/main/java/org/apache/fory/memory/MemoryBuffer.java b/java/fory-core/src/main/java/org/apache/fory/memory/MemoryBuffer.java index 743ccd32bc..6f5f8b5a8d 100644 --- a/java/fory-core/src/main/java/org/apache/fory/memory/MemoryBuffer.java +++ b/java/fory-core/src/main/java/org/apache/fory/memory/MemoryBuffer.java @@ -29,7 +29,8 @@ import org.apache.fory.io.AbstractStreamReader; import org.apache.fory.io.ForyStreamReader; import org.apache.fory.platform.AndroidSupport; -import org.apache.fory.platform.UnsafeOps; +import org.apache.fory.platform.JdkVersion; +import org.apache.fory.platform.internal._JDKAccess; import sun.misc.Unsafe; /** @@ -65,9 +66,29 @@ */ public final class MemoryBuffer { public static final int BUFFER_GROW_STEP_THRESHOLD = 100 * 1024 * 1024; - private static final Unsafe UNSAFE = AndroidSupport.IS_ANDROID ? null : UnsafeOps.UNSAFE; + private static final Unsafe UNSAFE = AndroidSupport.IS_ANDROID ? null : _JDKAccess.UNSAFE; private static final boolean LITTLE_ENDIAN = NativeByteOrder.IS_LITTLE_ENDIAN; - private static final boolean UNALIGNED = !AndroidSupport.IS_ANDROID && UnsafeOps.unaligned(); + private static final boolean UNALIGNED = !AndroidSupport.IS_ANDROID && unaligned(); + private static final int BOOLEAN_ARRAY_OFFSET = + AndroidSupport.IS_ANDROID ? 0 : UNSAFE.arrayBaseOffset(boolean[].class); + private static final int BYTE_ARRAY_OFFSET = + AndroidSupport.IS_ANDROID ? 0 : UNSAFE.arrayBaseOffset(byte[].class); + private static final int CHAR_ARRAY_OFFSET = + AndroidSupport.IS_ANDROID ? 0 : UNSAFE.arrayBaseOffset(char[].class); + private static final int SHORT_ARRAY_OFFSET = + AndroidSupport.IS_ANDROID ? 0 : UNSAFE.arrayBaseOffset(short[].class); + private static final int INT_ARRAY_OFFSET = + AndroidSupport.IS_ANDROID ? 0 : UNSAFE.arrayBaseOffset(int[].class); + private static final int LONG_ARRAY_OFFSET = + AndroidSupport.IS_ANDROID ? 0 : UNSAFE.arrayBaseOffset(long[].class); + private static final int FLOAT_ARRAY_OFFSET = + AndroidSupport.IS_ANDROID ? 0 : UNSAFE.arrayBaseOffset(float[].class); + private static final int DOUBLE_ARRAY_OFFSET = + AndroidSupport.IS_ANDROID ? 0 : UNSAFE.arrayBaseOffset(double[].class); + + /** Limits each raw Unsafe copy to let large copies hit safepoint polls between chunks. */ + private static final long UNSAFE_COPY_THRESHOLD = 1024L * 1024L; + // Global allocator instance that can be customized private static volatile MemoryAllocator globalAllocator = new DefaultMemoryAllocator(); @@ -77,7 +98,7 @@ private static final class DirectBufferAccess { static { try { Field addressField = Buffer.class.getDeclaredField("address"); - BUFFER_ADDRESS_FIELD_OFFSET = UnsafeOps.objectFieldOffset(addressField); + BUFFER_ADDRESS_FIELD_OFFSET = UNSAFE.objectFieldOffset(addressField); checkArgument(BUFFER_ADDRESS_FIELD_OFFSET != 0); } catch (NoSuchFieldException e) { throw new IllegalStateException(e); @@ -85,6 +106,41 @@ private static final class DirectBufferAccess { } } + private static boolean unaligned() { + String arch = System.getProperty("os.arch", ""); + if ("ppc64le".equals(arch) || "ppc64".equals(arch) || "s390x".equals(arch)) { + return true; + } + try { + Class bitsClass = + Class.forName("java.nio.Bits", false, ClassLoader.getSystemClassLoader()); + if (JdkVersion.MAJOR_VERSION >= 9) { + Field unalignedField = + bitsClass.getDeclaredField(JdkVersion.MAJOR_VERSION >= 11 ? "UNALIGNED" : "unaligned"); + return UNSAFE.getBoolean( + UNSAFE.staticFieldBase(unalignedField), UNSAFE.staticFieldOffset(unalignedField)); + } + return Boolean.TRUE.equals(bitsClass.getDeclaredMethod("unaligned").invoke(null)); + } catch (Throwable t) { + return arch.matches("^(i[3-6]86|x86(_64)?|x64|amd64|aarch64)$"); + } + } + + private static void copyMemory( + Object src, long srcOffset, Object dst, long dstOffset, long length) { + if (length < UNSAFE_COPY_THRESHOLD) { + UNSAFE.copyMemory(src, srcOffset, dst, dstOffset, length); + } else { + while (length > 0) { + long size = Math.min(length, UNSAFE_COPY_THRESHOLD); + UNSAFE.copyMemory(src, srcOffset, dst, dstOffset, size); + length -= size; + srcOffset += size; + dstOffset += size; + } + } + } + // If the data in on the heap, `heapMemory` will be non-null, and its' the object relative to // which we access the memory. // If we have this buffer, we must never void this reference, or the memory buffer will point @@ -208,7 +264,7 @@ private static long getAddress(ByteBuffer buffer) { checkNotNull(buffer, "buffer is null"); checkArgument(buffer.isDirect(), "Can't get address of a non-direct ByteBuffer."); try { - return UnsafeOps.getLong(buffer, DirectBufferAccess.BUFFER_ADDRESS_FIELD_OFFSET); + return UNSAFE.getLong(buffer, DirectBufferAccess.BUFFER_ADDRESS_FIELD_OFFSET); } catch (Throwable t) { throw new Error("Could not access direct byte buffer address field.", t); } @@ -268,7 +324,7 @@ public void initHeapBuffer(byte[] buffer, int offset, int length) { } this.heapMemory = buffer; this.heapOffset = offset; - final long startPos = UnsafeOps.BYTE_ARRAY_OFFSET + offset; + final long startPos = BYTE_ARRAY_OFFSET + offset; this.address = startPos; this.size = length; this.addressLimit = startPos + length; @@ -379,7 +435,7 @@ public void get(int index, byte[] dst, int offset, int length) { < 0) { throwOOBException(); } - UnsafeOps.copyMemory(null, pos, dst, UnsafeOps.BYTE_ARRAY_OFFSET + offset, length); + copyMemory(null, pos, dst, BYTE_ARRAY_OFFSET + offset, length); } } @@ -400,7 +456,7 @@ public void get(int offset, ByteBuffer target, int numBytes) { final long targetAddr = getAddress(target) + targetPos; final long sourceAddr = address + offset; if (sourceAddr <= addressLimit - numBytes) { - UnsafeOps.copyMemory(heapMemory, sourceAddr, null, targetAddr, numBytes); + copyMemory(heapMemory, sourceAddr, null, targetAddr, numBytes); } else { throwOOBException(); } @@ -425,7 +481,7 @@ public void put(int offset, ByteBuffer source, int numBytes) { final long sourceAddr = getAddress(source) + sourcePos; final long targetAddr = address + offset; if (targetAddr <= addressLimit - numBytes) { - UnsafeOps.copyMemory(null, sourceAddr, heapMemory, targetAddr, numBytes); + copyMemory(null, sourceAddr, heapMemory, targetAddr, numBytes); } else { throwOOBException(); } @@ -459,8 +515,8 @@ public void put(int index, byte[] src, int offset, int length) { < 0) { throwOOBException(); } - final long arrayAddress = UnsafeOps.BYTE_ARRAY_OFFSET + offset; - UnsafeOps.copyMemory(src, arrayAddress, null, pos, length); + final long arrayAddress = BYTE_ARRAY_OFFSET + offset; + copyMemory(src, arrayAddress, null, pos, length); } } @@ -1688,12 +1744,8 @@ public void writeBooleans(boolean[] values, int offset, int numElements) { final int writerIdx = writerIndex; final int newIdx = writerIdx + numElements; ensure(newIdx); - UNSAFE.copyMemory( - values, - UnsafeOps.BOOLEAN_ARRAY_OFFSET + offset, - heapMemory, - address + writerIdx, - numElements); + copyMemory( + values, BOOLEAN_ARRAY_OFFSET + offset, heapMemory, address + writerIdx, numElements); writerIndex = newIdx; } } @@ -1720,9 +1772,9 @@ public void writeChars(char[] values, int offset, int numElements) { final int writerIdx = writerIndex; final int newIdx = writerIdx + numBytes; ensure(newIdx); - UNSAFE.copyMemory( + copyMemory( values, - UnsafeOps.CHAR_ARRAY_OFFSET + ((long) offset << 1), + CHAR_ARRAY_OFFSET + ((long) offset << 1), heapMemory, address + writerIdx, numBytes); @@ -1752,9 +1804,9 @@ public void writeShorts(short[] values, int offset, int numElements) { final int writerIdx = writerIndex; final int newIdx = writerIdx + numBytes; ensure(newIdx); - UNSAFE.copyMemory( + copyMemory( values, - UnsafeOps.SHORT_ARRAY_OFFSET + ((long) offset << 1), + SHORT_ARRAY_OFFSET + ((long) offset << 1), heapMemory, address + writerIdx, numBytes); @@ -1784,9 +1836,9 @@ public void writeInts(int[] values, int offset, int numElements) { final int writerIdx = writerIndex; final int newIdx = writerIdx + numBytes; ensure(newIdx); - UNSAFE.copyMemory( + copyMemory( values, - UnsafeOps.INT_ARRAY_OFFSET + ((long) offset << 2), + INT_ARRAY_OFFSET + ((long) offset << 2), heapMemory, address + writerIdx, numBytes); @@ -1816,9 +1868,9 @@ public void writeLongs(long[] values, int offset, int numElements) { final int writerIdx = writerIndex; final int newIdx = writerIdx + numBytes; ensure(newIdx); - UNSAFE.copyMemory( + copyMemory( values, - UnsafeOps.LONG_ARRAY_OFFSET + ((long) offset << 3), + LONG_ARRAY_OFFSET + ((long) offset << 3), heapMemory, address + writerIdx, numBytes); @@ -1848,9 +1900,9 @@ public void writeFloats(float[] values, int offset, int numElements) { final int writerIdx = writerIndex; final int newIdx = writerIdx + numBytes; ensure(newIdx); - UNSAFE.copyMemory( + copyMemory( values, - UnsafeOps.FLOAT_ARRAY_OFFSET + ((long) offset << 2), + FLOAT_ARRAY_OFFSET + ((long) offset << 2), heapMemory, address + writerIdx, numBytes); @@ -1880,9 +1932,9 @@ public void writeDoubles(double[] values, int offset, int numElements) { final int writerIdx = writerIndex; final int newIdx = writerIdx + numBytes; ensure(newIdx); - UNSAFE.copyMemory( + copyMemory( values, - UnsafeOps.DOUBLE_ARRAY_OFFSET + ((long) offset << 3), + DOUBLE_ARRAY_OFFSET + ((long) offset << 3), heapMemory, address + writerIdx, numBytes); @@ -3056,7 +3108,7 @@ public byte[] readBytes(int length) { // System.arraycopy faster for some jdk than Unsafe. System.arraycopy(heapMemory, heapOffset + readerIdx, bytes, 0, length); } else { - UnsafeOps.copyMemory(null, address + readerIdx, bytes, UnsafeOps.BYTE_ARRAY_OFFSET, length); + copyMemory(null, address + readerIdx, bytes, BYTE_ARRAY_OFFSET, length); } readerIndex = readerIdx + length; return bytes; @@ -3226,8 +3278,7 @@ public byte[] readBytesAndSize() { if (heapMemory != null) { System.arraycopy(heapMemory, heapOffset + readerIdx, arr, 0, numBytes); } else { - UnsafeOps.UNSAFE.copyMemory( - null, address + readerIdx, arr, UnsafeOps.BYTE_ARRAY_OFFSET, numBytes); + copyMemory(null, address + readerIdx, arr, BYTE_ARRAY_OFFSET, numBytes); } readerIndex = readerIdx + numBytes; return arr; @@ -3251,7 +3302,7 @@ public void readByteArrayPayload(byte[] values, int numBytes) { if (heapMemory != null) { System.arraycopy(heapMemory, heapOffset + readerIdx, values, 0, numBytes); } else { - UNSAFE.copyMemory(null, address + readerIdx, values, UnsafeOps.BYTE_ARRAY_OFFSET, numBytes); + copyMemory(null, address + readerIdx, values, BYTE_ARRAY_OFFSET, numBytes); } readerIndex = readerIdx + numBytes; } @@ -3271,8 +3322,7 @@ public void readBooleanArrayPayload(boolean[] values, int numBytes) { streamReader.readBooleans(values, 0, numBytes); return; } - UNSAFE.copyMemory( - heapMemory, address + readerIdx, values, UnsafeOps.BOOLEAN_ARRAY_OFFSET, numBytes); + copyMemory(heapMemory, address + readerIdx, values, BOOLEAN_ARRAY_OFFSET, numBytes); readerIndex = readerIdx + numBytes; } } @@ -3291,8 +3341,7 @@ public void readCharArrayPayload(char[] values, int numBytes) { streamReader.readChars(values, 0, numBytes >>> 1); return; } - UNSAFE.copyMemory( - heapMemory, address + readerIdx, values, UnsafeOps.CHAR_ARRAY_OFFSET, numBytes); + copyMemory(heapMemory, address + readerIdx, values, CHAR_ARRAY_OFFSET, numBytes); readerIndex = readerIdx + numBytes; } } @@ -3311,8 +3360,7 @@ public void readInt16ArrayPayload(short[] values, int numBytes) { streamReader.readShorts(values, 0, numBytes >>> 1); return; } - UNSAFE.copyMemory( - heapMemory, address + readerIdx, values, UnsafeOps.SHORT_ARRAY_OFFSET, numBytes); + copyMemory(heapMemory, address + readerIdx, values, SHORT_ARRAY_OFFSET, numBytes); readerIndex = readerIdx + numBytes; } } @@ -3331,8 +3379,7 @@ public void readInt32ArrayPayload(int[] values, int numBytes) { streamReader.readInts(values, 0, numBytes >>> 2); return; } - UNSAFE.copyMemory( - heapMemory, address + readerIdx, values, UnsafeOps.INT_ARRAY_OFFSET, numBytes); + copyMemory(heapMemory, address + readerIdx, values, INT_ARRAY_OFFSET, numBytes); readerIndex = readerIdx + numBytes; } } @@ -3351,8 +3398,7 @@ public void readInt64ArrayPayload(long[] values, int numBytes) { streamReader.readLongs(values, 0, numBytes >>> 3); return; } - UNSAFE.copyMemory( - heapMemory, address + readerIdx, values, UnsafeOps.LONG_ARRAY_OFFSET, numBytes); + copyMemory(heapMemory, address + readerIdx, values, LONG_ARRAY_OFFSET, numBytes); readerIndex = readerIdx + numBytes; } } @@ -3371,8 +3417,7 @@ public void readFloat32ArrayPayload(float[] values, int numBytes) { streamReader.readFloats(values, 0, numBytes >>> 2); return; } - UNSAFE.copyMemory( - heapMemory, address + readerIdx, values, UnsafeOps.FLOAT_ARRAY_OFFSET, numBytes); + copyMemory(heapMemory, address + readerIdx, values, FLOAT_ARRAY_OFFSET, numBytes); readerIndex = readerIdx + numBytes; } } @@ -3391,8 +3436,7 @@ public void readFloat64ArrayPayload(double[] values, int numBytes) { streamReader.readDoubles(values, 0, numBytes >>> 3); return; } - UNSAFE.copyMemory( - heapMemory, address + readerIdx, values, UnsafeOps.DOUBLE_ARRAY_OFFSET, numBytes); + copyMemory(heapMemory, address + readerIdx, values, DOUBLE_ARRAY_OFFSET, numBytes); readerIndex = readerIdx + numBytes; } } @@ -3410,12 +3454,8 @@ public void readBooleans(boolean[] values, int offset, int numElements) { return; } int readerIdx = readerIndex; - UNSAFE.copyMemory( - heapMemory, - address + readerIdx, - values, - UnsafeOps.BOOLEAN_ARRAY_OFFSET + offset, - numElements); + copyMemory( + heapMemory, address + readerIdx, values, BOOLEAN_ARRAY_OFFSET + offset, numElements); readerIndex = readerIdx + numElements; } } @@ -3438,11 +3478,11 @@ public void readChars(char[] chars, int offset, int numElements) { return; } int readerIdx = readerIndex; - UNSAFE.copyMemory( + copyMemory( heapMemory, address + readerIdx, chars, - UnsafeOps.CHAR_ARRAY_OFFSET + ((long) offset << 1), + CHAR_ARRAY_OFFSET + ((long) offset << 1), numBytes); readerIndex = readerIdx + numBytes; } @@ -3471,11 +3511,11 @@ public void readShorts(short[] values, int offset, int numElements) { return; } int readerIdx = readerIndex; - UNSAFE.copyMemory( + copyMemory( heapMemory, address + readerIdx, values, - UnsafeOps.SHORT_ARRAY_OFFSET + ((long) offset << 1), + SHORT_ARRAY_OFFSET + ((long) offset << 1), numBytes); readerIndex = readerIdx + numBytes; } @@ -3495,11 +3535,11 @@ public void readInts(int[] values, int offset, int numElements) { return; } int readerIdx = readerIndex; - UNSAFE.copyMemory( + copyMemory( heapMemory, address + readerIdx, values, - UnsafeOps.INT_ARRAY_OFFSET + ((long) offset << 2), + INT_ARRAY_OFFSET + ((long) offset << 2), numBytes); readerIndex = readerIdx + numBytes; } @@ -3519,11 +3559,11 @@ public void readLongs(long[] values, int offset, int numElements) { return; } int readerIdx = readerIndex; - UNSAFE.copyMemory( + copyMemory( heapMemory, address + readerIdx, values, - UnsafeOps.LONG_ARRAY_OFFSET + ((long) offset << 3), + LONG_ARRAY_OFFSET + ((long) offset << 3), numBytes); readerIndex = readerIdx + numBytes; } @@ -3543,11 +3583,11 @@ public void readFloats(float[] values, int offset, int numElements) { return; } int readerIdx = readerIndex; - UNSAFE.copyMemory( + copyMemory( heapMemory, address + readerIdx, values, - UnsafeOps.FLOAT_ARRAY_OFFSET + ((long) offset << 2), + FLOAT_ARRAY_OFFSET + ((long) offset << 2), numBytes); readerIndex = readerIdx + numBytes; } @@ -3567,11 +3607,11 @@ public void readDoubles(double[] values, int offset, int numElements) { return; } int readerIdx = readerIndex; - UNSAFE.copyMemory( + copyMemory( heapMemory, address + readerIdx, values, - UnsafeOps.DOUBLE_ARRAY_OFFSET + ((long) offset << 3), + DOUBLE_ARRAY_OFFSET + ((long) offset << 3), numBytes); readerIndex = readerIdx + numBytes; } @@ -3611,7 +3651,7 @@ public void copyTo(int offset, MemoryBuffer target, int targetOffset, int numByt if ((numBytes | offset | targetOffset) >= 0 && thisPointer <= this.addressLimit - numBytes && otherPointer <= target.addressLimit - numBytes) { - UNSAFE.copyMemory(thisHeapRef, thisPointer, otherHeapRef, otherPointer, numBytes); + copyMemory(thisHeapRef, thisPointer, otherHeapRef, otherPointer, numBytes); } else { throw new IndexOutOfBoundsException( String.format( @@ -3630,12 +3670,7 @@ public void copyToByteArray(int offset, byte[] target, int targetOffset, int num MemoryOps.copyToByteArray(this, offset, target, targetOffset, numBytes); } else { checkArrayCopy(offset, targetOffset, target.length, numBytes, 0); - UNSAFE.copyMemory( - heapMemory, - address + offset, - target, - UnsafeOps.BYTE_ARRAY_OFFSET + targetOffset, - numBytes); + copyMemory(heapMemory, address + offset, target, BYTE_ARRAY_OFFSET + targetOffset, numBytes); } } @@ -3644,12 +3679,8 @@ public void copyToBooleanArray(int offset, boolean[] target, int targetOffset, i MemoryOps.copyToBooleanArray(this, offset, target, targetOffset, numBytes); } else { checkArrayCopy(offset, targetOffset, target.length, numBytes, 0); - UNSAFE.copyMemory( - heapMemory, - address + offset, - target, - UnsafeOps.BOOLEAN_ARRAY_OFFSET + targetOffset, - numBytes); + copyMemory( + heapMemory, address + offset, target, BOOLEAN_ARRAY_OFFSET + targetOffset, numBytes); } } @@ -3658,11 +3689,11 @@ public void copyToCharArray(int offset, char[] target, int targetOffset, int num MemoryOps.copyToCharArray(this, offset, target, targetOffset, numBytes); } else { checkArrayCopy(offset, targetOffset, target.length, numBytes, 1); - UNSAFE.copyMemory( + copyMemory( heapMemory, address + offset, target, - UnsafeOps.CHAR_ARRAY_OFFSET + arrayCopyOffset(targetOffset, 1), + CHAR_ARRAY_OFFSET + arrayCopyOffset(targetOffset, 1), numBytes); } } @@ -3672,11 +3703,11 @@ public void copyToShortArray(int offset, short[] target, int targetOffset, int n MemoryOps.copyToShortArray(this, offset, target, targetOffset, numBytes); } else { checkArrayCopy(offset, targetOffset, target.length, numBytes, 1); - UNSAFE.copyMemory( + copyMemory( heapMemory, address + offset, target, - UnsafeOps.SHORT_ARRAY_OFFSET + arrayCopyOffset(targetOffset, 1), + SHORT_ARRAY_OFFSET + arrayCopyOffset(targetOffset, 1), numBytes); } } @@ -3686,11 +3717,11 @@ public void copyToIntArray(int offset, int[] target, int targetOffset, int numBy MemoryOps.copyToIntArray(this, offset, target, targetOffset, numBytes); } else { checkArrayCopy(offset, targetOffset, target.length, numBytes, 2); - UNSAFE.copyMemory( + copyMemory( heapMemory, address + offset, target, - UnsafeOps.INT_ARRAY_OFFSET + arrayCopyOffset(targetOffset, 2), + INT_ARRAY_OFFSET + arrayCopyOffset(targetOffset, 2), numBytes); } } @@ -3700,11 +3731,11 @@ public void copyToLongArray(int offset, long[] target, int targetOffset, int num MemoryOps.copyToLongArray(this, offset, target, targetOffset, numBytes); } else { checkArrayCopy(offset, targetOffset, target.length, numBytes, 3); - UNSAFE.copyMemory( + copyMemory( heapMemory, address + offset, target, - UnsafeOps.LONG_ARRAY_OFFSET + arrayCopyOffset(targetOffset, 3), + LONG_ARRAY_OFFSET + arrayCopyOffset(targetOffset, 3), numBytes); } } @@ -3714,11 +3745,11 @@ public void copyToFloatArray(int offset, float[] target, int targetOffset, int n MemoryOps.copyToFloatArray(this, offset, target, targetOffset, numBytes); } else { checkArrayCopy(offset, targetOffset, target.length, numBytes, 2); - UNSAFE.copyMemory( + copyMemory( heapMemory, address + offset, target, - UnsafeOps.FLOAT_ARRAY_OFFSET + arrayCopyOffset(targetOffset, 2), + FLOAT_ARRAY_OFFSET + arrayCopyOffset(targetOffset, 2), numBytes); } } @@ -3728,11 +3759,114 @@ public void copyToDoubleArray(int offset, double[] target, int targetOffset, int MemoryOps.copyToDoubleArray(this, offset, target, targetOffset, numBytes); } else { checkArrayCopy(offset, targetOffset, target.length, numBytes, 3); - UNSAFE.copyMemory( + copyMemory( heapMemory, address + offset, target, - UnsafeOps.DOUBLE_ARRAY_OFFSET + arrayCopyOffset(targetOffset, 3), + DOUBLE_ARRAY_OFFSET + arrayCopyOffset(targetOffset, 3), + numBytes); + } + } + + public void copyFromByteArray(int offset, byte[] source, int sourceOffset, int numBytes) { + if (AndroidSupport.IS_ANDROID) { + MemoryOps.copyFromByteArray(this, offset, source, sourceOffset, numBytes); + } else { + checkArrayCopy(offset, sourceOffset, source.length, numBytes, 0); + copyMemory(source, BYTE_ARRAY_OFFSET + sourceOffset, heapMemory, address + offset, numBytes); + } + } + + public void copyFromBooleanArray(int offset, boolean[] source, int sourceOffset, int numBytes) { + if (AndroidSupport.IS_ANDROID) { + MemoryOps.copyFromBooleanArray(this, offset, source, sourceOffset, numBytes); + } else { + checkArrayCopy(offset, sourceOffset, source.length, numBytes, 0); + copyMemory( + source, BOOLEAN_ARRAY_OFFSET + sourceOffset, heapMemory, address + offset, numBytes); + } + } + + public void copyFromCharArray(int offset, char[] source, int sourceOffset, int numBytes) { + if (AndroidSupport.IS_ANDROID) { + MemoryOps.copyFromCharArray(this, offset, source, sourceOffset, numBytes); + } else { + checkArrayCopy(offset, sourceOffset, source.length, numBytes, 1); + copyMemory( + source, + CHAR_ARRAY_OFFSET + arrayCopyOffset(sourceOffset, 1), + heapMemory, + address + offset, + numBytes); + } + } + + public void copyFromShortArray(int offset, short[] source, int sourceOffset, int numBytes) { + if (AndroidSupport.IS_ANDROID) { + MemoryOps.copyFromShortArray(this, offset, source, sourceOffset, numBytes); + } else { + checkArrayCopy(offset, sourceOffset, source.length, numBytes, 1); + copyMemory( + source, + SHORT_ARRAY_OFFSET + arrayCopyOffset(sourceOffset, 1), + heapMemory, + address + offset, + numBytes); + } + } + + public void copyFromIntArray(int offset, int[] source, int sourceOffset, int numBytes) { + if (AndroidSupport.IS_ANDROID) { + MemoryOps.copyFromIntArray(this, offset, source, sourceOffset, numBytes); + } else { + checkArrayCopy(offset, sourceOffset, source.length, numBytes, 2); + copyMemory( + source, + INT_ARRAY_OFFSET + arrayCopyOffset(sourceOffset, 2), + heapMemory, + address + offset, + numBytes); + } + } + + public void copyFromLongArray(int offset, long[] source, int sourceOffset, int numBytes) { + if (AndroidSupport.IS_ANDROID) { + MemoryOps.copyFromLongArray(this, offset, source, sourceOffset, numBytes); + } else { + checkArrayCopy(offset, sourceOffset, source.length, numBytes, 3); + copyMemory( + source, + LONG_ARRAY_OFFSET + arrayCopyOffset(sourceOffset, 3), + heapMemory, + address + offset, + numBytes); + } + } + + public void copyFromFloatArray(int offset, float[] source, int sourceOffset, int numBytes) { + if (AndroidSupport.IS_ANDROID) { + MemoryOps.copyFromFloatArray(this, offset, source, sourceOffset, numBytes); + } else { + checkArrayCopy(offset, sourceOffset, source.length, numBytes, 2); + copyMemory( + source, + FLOAT_ARRAY_OFFSET + arrayCopyOffset(sourceOffset, 2), + heapMemory, + address + offset, + numBytes); + } + } + + public void copyFromDoubleArray(int offset, double[] source, int sourceOffset, int numBytes) { + if (AndroidSupport.IS_ANDROID) { + MemoryOps.copyFromDoubleArray(this, offset, source, sourceOffset, numBytes); + } else { + checkArrayCopy(offset, sourceOffset, source.length, numBytes, 3); + copyMemory( + source, + DOUBLE_ARRAY_OFFSET + arrayCopyOffset(sourceOffset, 3), + heapMemory, + address + offset, numBytes); } } @@ -3755,20 +3889,6 @@ private static long arrayCopyOffset(int elementOffset, int elementShift) { return (long) elementOffset << elementShift; } - /** - * JVM-only bulk copy method. Copies {@code numBytes} bytes from source unsafe object and pointer. - * Throws on Android before executing unsafe memory access. - */ - public void copyFromUnsafe(long offset, Object source, long sourcePointer, long numBytes) { - if (AndroidSupport.IS_ANDROID) { - MemoryOps.throwRawUnsafeMemoryCopyUnsupported(); - } else { - final long thisPointer = this.address + offset; - checkArgument(thisPointer + numBytes <= addressLimit); - UNSAFE.copyMemory(source, sourcePointer, heapMemory, thisPointer, numBytes); - } - } - public byte[] getBytes(int index, int length) { if (index == 0 && heapMemory != null && heapOffset == 0) { // Arrays.copyOf is an intrinsics, which is faster @@ -3884,7 +4004,7 @@ public boolean equalTo(byte[] bytes, int bytesOffset, int offset, int len) { return MemoryOps.equalTo(this, bytes, bytesOffset, offset, len); } final long pos = address + offset; - return unsafeEqualTo(heapMemory, pos, bytes, UnsafeOps.BYTE_ARRAY_OFFSET + bytesOffset, len); + return unsafeEqualTo(heapMemory, pos, bytes, BYTE_ARRAY_OFFSET + bytesOffset, len); } private static boolean unsafeEqualTo( diff --git a/java/fory-core/src/main/java/org/apache/fory/memory/MemoryOps.java b/java/fory-core/src/main/java/org/apache/fory/memory/MemoryOps.java index 8b1879f250..02722024ac 100644 --- a/java/fory-core/src/main/java/org/apache/fory/memory/MemoryOps.java +++ b/java/fory-core/src/main/java/org/apache/fory/memory/MemoryOps.java @@ -70,10 +70,6 @@ static MemoryBuffer directByteBufferUnsupported() { throw new UnsupportedOperationException("Direct ByteBuffer memory is not supported on Android"); } - static void throwRawUnsafeMemoryCopyUnsupported() { - throw new UnsupportedOperationException("Raw unsafe memory copy is not supported on Android"); - } - static MemoryBuffer fromByteBuffer(ByteBuffer buffer) { ByteBuffer duplicate = buffer.duplicate(); byte[] bytes = new byte[duplicate.remaining()]; @@ -1341,6 +1337,88 @@ static void copyToDoubleArray( } } + static void copyFromByteArray( + MemoryBuffer target, int offset, byte[] source, int sourceOffset, int numBytes) { + checkArrayCopy(target, offset, sourceOffset, source.length, numBytes, 0); + copy(source, sourceOffset, target.heapMemory, heapIndex(target, offset), numBytes); + } + + static void copyFromBooleanArray( + MemoryBuffer target, int offset, boolean[] source, int sourceOffset, int numBytes) { + checkArrayCopy(target, offset, sourceOffset, source.length, numBytes, 0); + byte[] bytes = target.heapMemory; + int targetIndex = heapIndex(target, offset); + for (int i = 0; i < numBytes; i++) { + bytes[targetIndex + i] = source[sourceOffset + i] ? (byte) 1 : (byte) 0; + } + } + + static void copyFromCharArray( + MemoryBuffer target, int offset, char[] source, int sourceOffset, int numBytes) { + checkArrayCopy(target, offset, sourceOffset, source.length, numBytes, 1); + byte[] bytes = target.heapMemory; + int targetIndex = heapIndex(target, offset); + int numElements = numBytes >>> 1; + for (int i = 0; i < numElements; i++, targetIndex += 2) { + putInt16(bytes, targetIndex, (short) source[sourceOffset + i]); + } + } + + static void copyFromShortArray( + MemoryBuffer target, int offset, short[] source, int sourceOffset, int numBytes) { + checkArrayCopy(target, offset, sourceOffset, source.length, numBytes, 1); + byte[] bytes = target.heapMemory; + int targetIndex = heapIndex(target, offset); + int numElements = numBytes >>> 1; + for (int i = 0; i < numElements; i++, targetIndex += 2) { + putInt16(bytes, targetIndex, source[sourceOffset + i]); + } + } + + static void copyFromIntArray( + MemoryBuffer target, int offset, int[] source, int sourceOffset, int numBytes) { + checkArrayCopy(target, offset, sourceOffset, source.length, numBytes, 2); + byte[] bytes = target.heapMemory; + int targetIndex = heapIndex(target, offset); + int numElements = numBytes >>> 2; + for (int i = 0; i < numElements; i++, targetIndex += 4) { + putInt32(bytes, targetIndex, source[sourceOffset + i]); + } + } + + static void copyFromLongArray( + MemoryBuffer target, int offset, long[] source, int sourceOffset, int numBytes) { + checkArrayCopy(target, offset, sourceOffset, source.length, numBytes, 3); + byte[] bytes = target.heapMemory; + int targetIndex = heapIndex(target, offset); + int numElements = numBytes >>> 3; + for (int i = 0; i < numElements; i++, targetIndex += 8) { + putInt64(bytes, targetIndex, source[sourceOffset + i]); + } + } + + static void copyFromFloatArray( + MemoryBuffer target, int offset, float[] source, int sourceOffset, int numBytes) { + checkArrayCopy(target, offset, sourceOffset, source.length, numBytes, 2); + byte[] bytes = target.heapMemory; + int targetIndex = heapIndex(target, offset); + int numElements = numBytes >>> 2; + for (int i = 0; i < numElements; i++, targetIndex += 4) { + putFloat32(bytes, targetIndex, source[sourceOffset + i]); + } + } + + static void copyFromDoubleArray( + MemoryBuffer target, int offset, double[] source, int sourceOffset, int numBytes) { + checkArrayCopy(target, offset, sourceOffset, source.length, numBytes, 3); + byte[] bytes = target.heapMemory; + int targetIndex = heapIndex(target, offset); + int numElements = numBytes >>> 3; + for (int i = 0; i < numElements; i++, targetIndex += 8) { + putFloat64(bytes, targetIndex, source[sourceOffset + i]); + } + } + private static void checkArrayCopy( MemoryBuffer source, int offset, diff --git a/java/fory-core/src/main/java/org/apache/fory/platform/UnsafeOps.java b/java/fory-core/src/main/java/org/apache/fory/platform/UnsafeOps.java deleted file mode 100644 index 2dc5456a63..0000000000 --- a/java/fory-core/src/main/java/org/apache/fory/platform/UnsafeOps.java +++ /dev/null @@ -1,215 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.fory.platform; - -import java.lang.reflect.Field; -import java.lang.reflect.Method; -import org.apache.fory.annotation.Internal; -import org.apache.fory.platform.internal._JDKAccess; -import org.apache.fory.util.ExceptionUtils; -import sun.misc.Unsafe; - -// Derived from Apache Spark's unsafe memory utility. - -/** A utility class for unsafe memory operations. */ -@Internal -@SuppressWarnings("restriction") -public final class UnsafeOps { - @SuppressWarnings("restriction") - public static final Unsafe UNSAFE = _JDKAccess.UNSAFE; - - public static final int BOOLEAN_ARRAY_OFFSET; - public static final int BYTE_ARRAY_OFFSET; - public static final int CHAR_ARRAY_OFFSET; - public static final int SHORT_ARRAY_OFFSET; - public static final int INT_ARRAY_OFFSET; - public static final int LONG_ARRAY_OFFSET; - public static final int FLOAT_ARRAY_OFFSET; - public static final int DOUBLE_ARRAY_OFFSET; - private static final boolean unaligned; - - /** - * Limits the number of bytes to copy per {@link Unsafe#copyMemory(long, long, long)} to allow - * safepoint polling during a large copy. - */ - private static final long UNSAFE_COPY_THRESHOLD = 1024L * 1024L; - - private UnsafeOps() {} - - static { - BOOLEAN_ARRAY_OFFSET = UNSAFE.arrayBaseOffset(boolean[].class); - BYTE_ARRAY_OFFSET = UNSAFE.arrayBaseOffset(byte[].class); - CHAR_ARRAY_OFFSET = UNSAFE.arrayBaseOffset(char[].class); - SHORT_ARRAY_OFFSET = UNSAFE.arrayBaseOffset(short[].class); - INT_ARRAY_OFFSET = UNSAFE.arrayBaseOffset(int[].class); - LONG_ARRAY_OFFSET = UNSAFE.arrayBaseOffset(long[].class); - FLOAT_ARRAY_OFFSET = UNSAFE.arrayBaseOffset(float[].class); - DOUBLE_ARRAY_OFFSET = UNSAFE.arrayBaseOffset(double[].class); - } - - // This requires `JdkVersion.MAJOR_VERSION` and `_UNSAFE`. - static { - boolean unalign; - String arch = System.getProperty("os.arch", ""); - if ("ppc64le".equals(arch) || "ppc64".equals(arch) || "s390x".equals(arch)) { - // Since java.nio.Bits.unaligned() doesn't return true on ppc (See JDK-8165231), but - // ppc64 and ppc64le support it - unalign = true; - } else { - try { - Class bitsClass = - Class.forName("java.nio.Bits", false, ClassLoader.getSystemClassLoader()); - if (JdkVersion.MAJOR_VERSION >= 9) { - // Java 9/10 and 11/12 have different field names. - Field unalignedField = - bitsClass.getDeclaredField( - JdkVersion.MAJOR_VERSION >= 11 ? "UNALIGNED" : "unaligned"); - unalign = - UNSAFE.getBoolean( - UNSAFE.staticFieldBase(unalignedField), UNSAFE.staticFieldOffset(unalignedField)); - } else { - Method unalignedMethod = bitsClass.getDeclaredMethod("unaligned"); - unalignedMethod.setAccessible(true); - unalign = Boolean.TRUE.equals(unalignedMethod.invoke(null)); - } - } catch (Throwable t) { - // We at least know x86 and x64 support unaligned access. - //noinspection DynamicRegexReplaceableByCompiledPattern - unalign = arch.matches("^(i[3-6]86|x86(_64)?|x64|amd64|aarch64)$"); - } - } - unaligned = unalign; - } - - /** - * Returns true when running JVM is having sun's Unsafe package available in it and underlying - * system having unaligned-access capability. - */ - public static boolean unaligned() { - return unaligned; - } - - public static long objectFieldOffset(Field f) { - return UNSAFE.objectFieldOffset(f); - } - - public static int getInt(Object object, long offset) { - return UNSAFE.getInt(object, offset); - } - - public static void putInt(Object object, long offset, int value) { - UNSAFE.putInt(object, offset, value); - } - - public static boolean getBoolean(Object object, long offset) { - return UNSAFE.getBoolean(object, offset); - } - - public static void putBoolean(Object object, long offset, boolean value) { - UNSAFE.putBoolean(object, offset, value); - } - - public static byte getByte(Object object, long offset) { - return UNSAFE.getByte(object, offset); - } - - public static void putByte(Object object, long offset, byte value) { - UNSAFE.putByte(object, offset, value); - } - - public static short getShort(Object object, long offset) { - return UNSAFE.getShort(object, offset); - } - - public static void putShort(Object object, long offset, short value) { - UNSAFE.putShort(object, offset, value); - } - - public static char getChar(Object obj, long offset) { - return UnsafeOps.UNSAFE.getChar(obj, offset); - } - - public static void putChar(Object obj, long offset, char value) { - UnsafeOps.UNSAFE.putChar(obj, offset, value); - } - - public static long getLong(Object object, long offset) { - return UNSAFE.getLong(object, offset); - } - - public static void putLong(Object object, long offset, long value) { - UNSAFE.putLong(object, offset, value); - } - - public static float getFloat(Object object, long offset) { - return UNSAFE.getFloat(object, offset); - } - - public static void putFloat(Object object, long offset, float value) { - UNSAFE.putFloat(object, offset, value); - } - - public static double getDouble(Object object, long offset) { - return UNSAFE.getDouble(object, offset); - } - - public static void putDouble(Object object, long offset, double value) { - UNSAFE.putDouble(object, offset, value); - } - - public static Object getObject(Object o, long offset) { - return UNSAFE.getObject(o, offset); - } - - public static void putObject(Object object, long offset, Object value) { - UNSAFE.putObject(object, offset, value); - } - - public static void copyMemory( - Object src, long srcOffset, Object dst, long dstOffset, long length) { - if (length < UNSAFE_COPY_THRESHOLD) { - UNSAFE.copyMemory(src, srcOffset, dst, dstOffset, length); - } else { - while (length > 0) { - long size = Math.min(length, UNSAFE_COPY_THRESHOLD); - UNSAFE.copyMemory(src, srcOffset, dst, dstOffset, size); - length -= size; - srcOffset += size; - dstOffset += size; - } - } - } - - public static Object[] copyObjectArray(Object[] arr) { - Object[] objects = new Object[arr.length]; - System.arraycopy(arr, 0, objects, 0, arr.length); - return objects; - } - - /** Create an instance of type. This method don't call constructor. */ - public static T newInstance(Class type) { - try { - return type.cast(UNSAFE.allocateInstance(type)); - } catch (InstantiationException e) { - ExceptionUtils.throwException(e); - } - throw new IllegalStateException("unreachable"); - } -} diff --git a/java/fory-core/src/main/java/org/apache/fory/platform/internal/_JDKAccess.java b/java/fory-core/src/main/java/org/apache/fory/platform/internal/_JDKAccess.java index 8584f722f4..9efba19593 100644 --- a/java/fory-core/src/main/java/org/apache/fory/platform/internal/_JDKAccess.java +++ b/java/fory-core/src/main/java/org/apache/fory/platform/internal/_JDKAccess.java @@ -124,6 +124,10 @@ public class _JDKAccess { _INNER_UNSAFE_CLASS = innerUnsafeClass; } + public static Unsafe unsafe() { + return UNSAFE; + } + private static final ClassValueCache lookupCache = ClassValueCache.newClassKeyCache(32); public static final boolean STRING_VALUE_FIELD_IS_CHARS; diff --git a/java/fory-core/src/main/java/org/apache/fory/reflect/FieldAccessor.java b/java/fory-core/src/main/java/org/apache/fory/reflect/FieldAccessor.java index 2eb540cb79..5aa6ebee5a 100644 --- a/java/fory-core/src/main/java/org/apache/fory/reflect/FieldAccessor.java +++ b/java/fory-core/src/main/java/org/apache/fory/reflect/FieldAccessor.java @@ -37,7 +37,6 @@ import org.apache.fory.exception.ForyException; import org.apache.fory.platform.AndroidSupport; import org.apache.fory.platform.GraalvmSupport; -import org.apache.fory.platform.UnsafeOps; import org.apache.fory.platform.internal._JDKAccess; import org.apache.fory.type.TypeUtils; import org.apache.fory.util.Preconditions; @@ -47,10 +46,12 @@ import org.apache.fory.util.function.ToFloatFunction; import org.apache.fory.util.function.ToShortFunction; import org.apache.fory.util.record.RecordUtils; +import sun.misc.Unsafe; /** Field accessor for primitive types and object types. */ @SuppressWarnings({"unchecked", "rawtypes"}) public abstract class FieldAccessor { + private static final Unsafe UNSAFE = AndroidSupport.IS_ANDROID ? null : _JDKAccess.UNSAFE; private static final int REFLECTIVE_ACCESS = 0; private static final int BOOLEAN_ACCESS = 1; private static final int BYTE_ACCESS = 2; @@ -81,7 +82,7 @@ private static long fieldOffset(Field field) { // Field offsets are rewritten by GraalVM and are not stable during native-image build time. return -1; } - return UnsafeOps.objectFieldOffset(field); + return UNSAFE.objectFieldOffset(field); } protected FieldAccessor(Field field, long fieldOffset) { @@ -124,36 +125,31 @@ public void set(Object obj, Object value) { public final void copy(Object sourceObject, Object targetObject) { switch (accessKind) { case BOOLEAN_ACCESS: - UnsafeOps.putBoolean( - targetObject, fieldOffset, UnsafeOps.getBoolean(sourceObject, fieldOffset)); + UNSAFE.putBoolean(targetObject, fieldOffset, UNSAFE.getBoolean(sourceObject, fieldOffset)); return; case BYTE_ACCESS: - UnsafeOps.putByte(targetObject, fieldOffset, UnsafeOps.getByte(sourceObject, fieldOffset)); + UNSAFE.putByte(targetObject, fieldOffset, UNSAFE.getByte(sourceObject, fieldOffset)); return; case CHAR_ACCESS: - UnsafeOps.putChar(targetObject, fieldOffset, UnsafeOps.getChar(sourceObject, fieldOffset)); + UNSAFE.putChar(targetObject, fieldOffset, UNSAFE.getChar(sourceObject, fieldOffset)); return; case SHORT_ACCESS: - UnsafeOps.putShort( - targetObject, fieldOffset, UnsafeOps.getShort(sourceObject, fieldOffset)); + UNSAFE.putShort(targetObject, fieldOffset, UNSAFE.getShort(sourceObject, fieldOffset)); return; case INT_ACCESS: - UnsafeOps.putInt(targetObject, fieldOffset, UnsafeOps.getInt(sourceObject, fieldOffset)); + UNSAFE.putInt(targetObject, fieldOffset, UNSAFE.getInt(sourceObject, fieldOffset)); return; case LONG_ACCESS: - UnsafeOps.putLong(targetObject, fieldOffset, UnsafeOps.getLong(sourceObject, fieldOffset)); + UNSAFE.putLong(targetObject, fieldOffset, UNSAFE.getLong(sourceObject, fieldOffset)); return; case FLOAT_ACCESS: - UnsafeOps.putFloat( - targetObject, fieldOffset, UnsafeOps.getFloat(sourceObject, fieldOffset)); + UNSAFE.putFloat(targetObject, fieldOffset, UNSAFE.getFloat(sourceObject, fieldOffset)); return; case DOUBLE_ACCESS: - UnsafeOps.putDouble( - targetObject, fieldOffset, UnsafeOps.getDouble(sourceObject, fieldOffset)); + UNSAFE.putDouble(targetObject, fieldOffset, UNSAFE.getDouble(sourceObject, fieldOffset)); return; case OBJECT_ACCESS: - UnsafeOps.putObject( - targetObject, fieldOffset, UnsafeOps.getObject(sourceObject, fieldOffset)); + UNSAFE.putObject(targetObject, fieldOffset, UNSAFE.getObject(sourceObject, fieldOffset)); return; default: putObject(targetObject, getObject(sourceObject)); @@ -162,8 +158,7 @@ public final void copy(Object sourceObject, Object targetObject) { public final void copyObject(Object sourceObject, Object targetObject) { if (accessKind == OBJECT_ACCESS) { - UnsafeOps.putObject( - targetObject, fieldOffset, UnsafeOps.getObject(sourceObject, fieldOffset)); + UNSAFE.putObject(targetObject, fieldOffset, UNSAFE.getObject(sourceObject, fieldOffset)); } else { putObject(targetObject, getObject(sourceObject)); } @@ -238,21 +233,21 @@ public void putDouble(Object targetObject, double value) { } public final void putObject(Object targetObject, Object object) { - // For primitive fields, we must use set() which calls the correct UnsafeOps.putXxx method. - // UnsafeOps.putObject writes object references, not primitive values. + // For primitive fields, we must use set() which calls the correct UNSAFE.putXxx method. + // UNSAFE.putObject writes object references, not primitive values. if (fieldOffset != -1 && !field.getType().isPrimitive()) { - UnsafeOps.putObject(targetObject, fieldOffset, object); + UNSAFE.putObject(targetObject, fieldOffset, object); } else { set(targetObject, object); } } public final Object getObject(Object targetObject) { - // For primitive fields, we must use get() which calls the correct UnsafeOps.getXxx method - // and returns the boxed value. UnsafeOps.getObject interprets primitive bytes as object + // For primitive fields, we must use get() which calls the correct UNSAFE.getXxx method + // and returns the boxed value. UNSAFE.getObject interprets primitive bytes as object // refs. if (fieldOffset != -1 && !field.getType().isPrimitive()) { - return UnsafeOps.getObject(targetObject, fieldOffset); + return UNSAFE.getObject(targetObject, fieldOffset); } else { return get(targetObject); } @@ -401,7 +396,7 @@ public Object get(Object obj) { @Override public boolean getBoolean(Object obj) { checkObj(obj); - return UnsafeOps.getBoolean(obj, fieldOffset); + return UNSAFE.getBoolean(obj, fieldOffset); } @Override @@ -412,7 +407,7 @@ public void set(Object obj, Object value) { @Override public void putBoolean(Object obj, boolean value) { checkObj(obj); - UnsafeOps.putBoolean(obj, fieldOffset, value); + UNSAFE.putBoolean(obj, fieldOffset, value); } } @@ -452,7 +447,7 @@ public Byte get(Object obj) { @Override public byte getByte(Object obj) { checkObj(obj); - return UnsafeOps.getByte(obj, fieldOffset); + return UNSAFE.getByte(obj, fieldOffset); } @Override @@ -463,7 +458,7 @@ public void set(Object obj, Object value) { @Override public void putByte(Object obj, byte value) { checkObj(obj); - UnsafeOps.putByte(obj, fieldOffset, value); + UNSAFE.putByte(obj, fieldOffset, value); } } @@ -503,7 +498,7 @@ public Character get(Object obj) { @Override public char getChar(Object obj) { checkObj(obj); - return UnsafeOps.getChar(obj, fieldOffset); + return UNSAFE.getChar(obj, fieldOffset); } @Override @@ -514,7 +509,7 @@ public void set(Object obj, Object value) { @Override public void putChar(Object obj, char value) { checkObj(obj); - UnsafeOps.putChar(obj, fieldOffset, value); + UNSAFE.putChar(obj, fieldOffset, value); } } @@ -553,7 +548,7 @@ public Short get(Object obj) { @Override public short getShort(Object obj) { checkObj(obj); - return UnsafeOps.getShort(obj, fieldOffset); + return UNSAFE.getShort(obj, fieldOffset); } @Override @@ -564,7 +559,7 @@ public void set(Object obj, Object value) { @Override public void putShort(Object obj, short value) { checkObj(obj); - UnsafeOps.putShort(obj, fieldOffset, value); + UNSAFE.putShort(obj, fieldOffset, value); } } @@ -603,7 +598,7 @@ public Integer get(Object obj) { @Override public int getInt(Object obj) { checkObj(obj); - return UnsafeOps.getInt(obj, fieldOffset); + return UNSAFE.getInt(obj, fieldOffset); } @Override @@ -614,7 +609,7 @@ public void set(Object obj, Object value) { @Override public void putInt(Object obj, int value) { checkObj(obj); - UnsafeOps.putInt(obj, fieldOffset, value); + UNSAFE.putInt(obj, fieldOffset, value); } } @@ -653,7 +648,7 @@ public Long get(Object obj) { @Override public long getLong(Object obj) { checkObj(obj); - return UnsafeOps.getLong(obj, fieldOffset); + return UNSAFE.getLong(obj, fieldOffset); } @Override @@ -664,7 +659,7 @@ public void set(Object obj, Object value) { @Override public void putLong(Object obj, long value) { checkObj(obj); - UnsafeOps.putLong(obj, fieldOffset, value); + UNSAFE.putLong(obj, fieldOffset, value); } } @@ -703,7 +698,7 @@ public Object get(Object obj) { @Override public float getFloat(Object obj) { checkObj(obj); - return UnsafeOps.getFloat(obj, fieldOffset); + return UNSAFE.getFloat(obj, fieldOffset); } @Override @@ -714,7 +709,7 @@ public void set(Object obj, Object value) { @Override public void putFloat(Object obj, float value) { checkObj(obj); - UnsafeOps.putFloat(obj, fieldOffset, value); + UNSAFE.putFloat(obj, fieldOffset, value); } } @@ -753,7 +748,7 @@ public Object get(Object obj) { @Override public double getDouble(Object obj) { checkObj(obj); - return UnsafeOps.getDouble(obj, fieldOffset); + return UNSAFE.getDouble(obj, fieldOffset); } @Override @@ -764,7 +759,7 @@ public void set(Object obj, Object value) { @Override public void putDouble(Object obj, double value) { checkObj(obj); - UnsafeOps.putDouble(obj, fieldOffset, value); + UNSAFE.putDouble(obj, fieldOffset, value); } } @@ -798,13 +793,13 @@ public ObjectAccessor(Field field) { @Override public Object get(Object obj) { checkObj(obj); - return UnsafeOps.getObject(obj, fieldOffset); + return UNSAFE.getObject(obj, fieldOffset); } @Override public void set(Object obj, Object value) { checkObj(obj); - UnsafeOps.putObject(obj, fieldOffset, value); + UNSAFE.putObject(obj, fieldOffset, value); } } @@ -854,18 +849,18 @@ static final class StaticObjectAccessor extends FieldAccessor { StaticObjectAccessor(Field field) { super(field, -1); Preconditions.checkArgument(!TypeUtils.isPrimitive(field.getType())); - base = UnsafeOps.UNSAFE.staticFieldBase(field); - offset = UnsafeOps.UNSAFE.staticFieldOffset(field); + base = UNSAFE.staticFieldBase(field); + offset = UNSAFE.staticFieldOffset(field); } @Override public Object get(Object obj) { - return UnsafeOps.getObject(base, offset); + return UNSAFE.getObject(base, offset); } @Override public void set(Object obj, Object value) { - UnsafeOps.putObject(base, offset, value); + UNSAFE.putObject(base, offset, value); } } diff --git a/java/fory-core/src/main/java/org/apache/fory/reflect/ObjectCreator.java b/java/fory-core/src/main/java/org/apache/fory/reflect/ObjectCreator.java index 75f8c64558..1134c21896 100644 --- a/java/fory-core/src/main/java/org/apache/fory/reflect/ObjectCreator.java +++ b/java/fory-core/src/main/java/org/apache/fory/reflect/ObjectCreator.java @@ -31,7 +31,7 @@ * *

Thread Safety: All implementations of ObjectCreator are thread-safe and can * be safely used across multiple threads concurrently. The underlying creation mechanisms - * (MethodHandle, Constructor, UnsafeOps.newInstance) are all thread-safe. + * (MethodHandle, Constructor, and supported constructor-bypassing allocation) are all thread-safe. * * @param the type of objects this creator can instantiate */ diff --git a/java/fory-core/src/main/java/org/apache/fory/reflect/ObjectCreators.java b/java/fory-core/src/main/java/org/apache/fory/reflect/ObjectCreators.java index 682a9546be..d365c4e578 100644 --- a/java/fory-core/src/main/java/org/apache/fory/reflect/ObjectCreators.java +++ b/java/fory-core/src/main/java/org/apache/fory/reflect/ObjectCreators.java @@ -40,11 +40,11 @@ import org.apache.fory.platform.AndroidSupport; import org.apache.fory.platform.GraalvmSupport; import org.apache.fory.platform.JdkVersion; -import org.apache.fory.platform.UnsafeOps; import org.apache.fory.platform.internal._JDKAccess; import org.apache.fory.type.Descriptor; import org.apache.fory.type.TypeUtils; import org.apache.fory.util.record.RecordUtils; +import sun.misc.Unsafe; /** * Factory class for creating and caching {@link ObjectCreator} instances. @@ -58,8 +58,8 @@ * parameterized constructor invocation *

  • Classes with no-arg constructors: Uses {@link * DeclaredNoArgCtrObjectCreator} with MethodHandle for fast invocation - *
  • Classes without accessible constructors: Uses {@link UnsafeObjectCreator} - * with platform-specific unsafe allocation + *
  • Classes without accessible constructors: Uses a private + * constructor-bypassing creator on runtimes where that is still supported *
  • GraalVM native image compatibility: Uses {@link * ParentNoArgCtrObjectCreator} for constructor generate-based creation when needed *
  • Android compatibility: Uses reflection for records and no-arg @@ -74,6 +74,7 @@ */ @SuppressWarnings("unchecked") public class ObjectCreators { + private static final Unsafe UNSAFE = AndroidSupport.IS_ANDROID ? null : _JDKAccess.UNSAFE; private static final ClassValueCache> cache = ClassValueCache.newClassKeySoftCache(8); @@ -94,6 +95,18 @@ public static ObjectCreator getObjectCreator(Class type) { return (ObjectCreator) cache.get(type, () -> creategetObjectCreator(type)); } + private static T allocateInstance(Class type) { + if (UNSAFE == null || JdkVersion.MAJOR_VERSION >= 25) { + throw new ForyException( + "Constructor-bypassing allocation is unsupported in this runtime for " + type); + } + try { + return (T) UNSAFE.allocateInstance(type); + } catch (InstantiationException e) { + throw new ForyException("Failed to allocate instance for " + type, e); + } + } + private static ObjectCreator creategetObjectCreator(Class type) { if (RecordUtils.isRecord(type)) { return new RecordObjectCreator<>(type); @@ -529,7 +542,7 @@ public T newInstanceWithArguments(Object... arguments) { } } - public static final class UnsafeObjectCreator extends ObjectCreator { + private static final class UnsafeObjectCreator extends ObjectCreator { public UnsafeObjectCreator(Class type) { super(type); @@ -537,7 +550,7 @@ public UnsafeObjectCreator(Class type) { @Override public T newInstance() { - return UnsafeOps.newInstance(type); + return ObjectCreators.allocateInstance(type); } @Override diff --git a/java/fory-core/src/main/java/org/apache/fory/reflect/ReflectionUtils.java b/java/fory-core/src/main/java/org/apache/fory/reflect/ReflectionUtils.java index 05f0e23712..82647de3ae 100644 --- a/java/fory-core/src/main/java/org/apache/fory/reflect/ReflectionUtils.java +++ b/java/fory-core/src/main/java/org/apache/fory/reflect/ReflectionUtils.java @@ -454,7 +454,7 @@ public static List getSortedFields(Class cls, boolean searchParent) { public static List getFieldValues(Collection fields, Object o) { List results = new ArrayList<>(fields.size()); for (Field field : fields) { - // UnsafeOps.objectFieldOffset(field) can't handle primitive field. + // Unsafe.objectFieldOffset(field) can't handle primitive field. Object fieldValue = FieldAccessor.createAccessor(field).get(o); results.add(fieldValue); } diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java index 24f3d7f472..bb2cb1a4cd 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java @@ -1004,7 +1004,6 @@ private boolean usesNonStructTypeDef(Class cls) { || isMap(cls) || Externalizable.class.isAssignableFrom(cls) || requireJavaSerialization(cls) - || requiresJavaSerializer(cls) || useReplaceResolveSerializer(cls) || Functions.isLambda(cls) || (ScalaTypes.SCALA_AVAILABLE && ReflectionUtils.isScalaSingletonObject(cls)) @@ -1546,9 +1545,6 @@ public Class getSerializerClass(Class cls, boolean code if (requiresJdkStream(cls)) { return getDefaultJDKStreamSerializerType(); } - if (requiresJavaSerializer(cls)) { - return JavaSerializer.class; - } if (isCrossLanguage()) { LOG.warn("Class {} isn't supported for cross-language serialization.", cls); } @@ -1646,18 +1642,6 @@ private static boolean requiresJdkStream(Class cls) { && !hasNoArgConstructor(cls); } - private static boolean requiresJavaSerializer(Class cls) { - if (JdkVersion.MAJOR_VERSION < 25 || !Serializable.class.isAssignableFrom(cls)) { - return false; - } - // Scala products can have final derived fields initialized by the primary constructor but not - // represented as constructor parameters. Keep that compatibility in the isolated JDK stream - // path instead of teaching the generic JDK25 field serializer to ignore final fields. - return ScalaTypes.SCALA_AVAILABLE - && ScalaTypes.isScalaProductType(cls) - && !ReflectionUtils.isScalaSingletonObject(cls); - } - private static boolean hasNoArgConstructor(Class cls) { try { cls.getDeclaredConstructor(); diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/CompatibleSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/CompatibleSerializer.java index d5e548fb45..01058fbd81 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/CompatibleSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/CompatibleSerializer.java @@ -34,8 +34,8 @@ import org.apache.fory.platform.AndroidSupport; import org.apache.fory.platform.GraalvmSupport; import org.apache.fory.platform.JdkVersion; -import org.apache.fory.platform.UnsafeOps; import org.apache.fory.reflect.FieldAccessor; +import org.apache.fory.reflect.ObjectCreators; import org.apache.fory.resolver.ClassResolver; import org.apache.fory.resolver.RefMode; import org.apache.fory.resolver.TypeResolver; @@ -267,7 +267,7 @@ private T newInstance() { || GraalvmSupport.IN_GRAALVM_NATIVE_IMAGE || JdkVersion.MAJOR_VERSION >= 25 ? newBean() - : UnsafeOps.newInstance(type); + : ObjectCreators.getObjectCreator(type).newInstance(); // Set default values for missing fields in Scala case classes DefaultValueUtils.setDefaultValues(obj, defaultValueFields); return obj; diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/ExceptionSerializers.java b/java/fory-core/src/main/java/org/apache/fory/serializer/ExceptionSerializers.java index 6cdeab3b7a..8c4286e852 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/ExceptionSerializers.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/ExceptionSerializers.java @@ -385,7 +385,7 @@ private static StackTraceElement newStackTraceElement( private static ObjectCreator createThrowableObjectCreator( Class type) { if (GraalvmSupport.IN_GRAALVM_NATIVE_IMAGE) { - return new ObjectCreators.UnsafeObjectCreator<>(type); + return ObjectCreators.getObjectCreator(type); } if (ReflectionUtils.getCtrHandle(type, false) != null) { return ObjectCreators.getObjectCreator(type); diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectStreamSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectStreamSerializer.java index 09b34f4318..e03dd8cdf2 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectStreamSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectStreamSerializer.java @@ -252,12 +252,8 @@ private static ObjectCreator createObjectStreamCreator(Class type) { // ObjectStreamSerializer must preserve Java serialization construction semantics. On JDK25+ // this path cannot fall back to Unsafe, including inside GraalVM native images. return new ObjectCreators.ParentNoArgCtrObjectCreator<>(type); - } else if (GraalvmSupport.IN_GRAALVM_NATIVE_IMAGE) { - return new ObjectCreators.UnsafeObjectCreator<>(type); - } else { - // In regular JVM, use the standard object creator - return ObjectCreators.getObjectCreator(type); } + return ObjectCreators.getObjectCreator(type); } private static boolean hasJdk25Fallback(Class type) { diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/PlatformStringUtils.java b/java/fory-core/src/main/java/org/apache/fory/serializer/PlatformStringUtils.java index 6ac082acdc..d9770e1892 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/PlatformStringUtils.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/PlatformStringUtils.java @@ -23,11 +23,17 @@ import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.platform.AndroidSupport; import org.apache.fory.platform.GraalvmSupport; -import org.apache.fory.platform.UnsafeOps; import org.apache.fory.platform.internal._JDKAccess; +import sun.misc.Unsafe; /** Platform-owned string internals used by {@link StringSerializer}. */ final class PlatformStringUtils { + private static final Unsafe UNSAFE = AndroidSupport.IS_ANDROID ? null : _JDKAccess.UNSAFE; + private static final int BYTE_ARRAY_OFFSET = + AndroidSupport.IS_ANDROID ? 0 : UNSAFE.arrayBaseOffset(byte[].class); + private static final int CHAR_ARRAY_OFFSET = + AndroidSupport.IS_ANDROID ? 0 : UNSAFE.arrayBaseOffset(char[].class); + static final boolean JDK_STRING_FIELD_ACCESS = !AndroidSupport.IS_ANDROID && !GraalvmSupport.IN_GRAALVM_NATIVE_IMAGE @@ -54,19 +60,19 @@ final class PlatformStringUtils { private PlatformStringUtils() {} static Object getStringValue(String value) { - return UnsafeOps.getObject(value, STRING_VALUE_FIELD_OFFSET); + return UNSAFE.getObject(value, STRING_VALUE_FIELD_OFFSET); } static byte getStringCoder(String value) { - return UnsafeOps.getByte(value, STRING_CODER_FIELD_OFFSET); + return UNSAFE.getByte(value, STRING_CODER_FIELD_OFFSET); } static int getStringOffset(String value) { - return UnsafeOps.getInt(value, STRING_OFFSET_FIELD_OFFSET); + return UNSAFE.getInt(value, STRING_OFFSET_FIELD_OFFSET); } static int getStringCount(String value) { - return UnsafeOps.getInt(value, STRING_COUNT_FIELD_OFFSET); + return UNSAFE.getInt(value, STRING_COUNT_FIELD_OFFSET); } static String newCharsStringZeroCopy(char[] data) { @@ -98,29 +104,29 @@ private static String newBytesStringSlow(byte coder, byte[] data) { } static long getCharsLong(char[] chars, int charIndex) { - return UnsafeOps.getLong(chars, UnsafeOps.CHAR_ARRAY_OFFSET + ((long) charIndex << 1)); + return UNSAFE.getLong(chars, CHAR_ARRAY_OFFSET + ((long) charIndex << 1)); } static long getBytesLong(byte[] bytes, int byteIndex) { - return UnsafeOps.getLong(bytes, UnsafeOps.BYTE_ARRAY_OFFSET + byteIndex); + return UNSAFE.getLong(bytes, BYTE_ARRAY_OFFSET + byteIndex); } static char getBytesChar(byte[] bytes, int byteIndex) { - return UnsafeOps.getChar(bytes, UnsafeOps.BYTE_ARRAY_OFFSET + byteIndex); + return UNSAFE.getChar(bytes, BYTE_ARRAY_OFFSET + byteIndex); } static void copyCharsToBytes( char[] chars, int charOffset, byte[] target, int byteOffset, int numBytes) { - UnsafeOps.UNSAFE.copyMemory( + UNSAFE.copyMemory( chars, - UnsafeOps.CHAR_ARRAY_OFFSET + ((long) charOffset << 1), + CHAR_ARRAY_OFFSET + ((long) charOffset << 1), target, - UnsafeOps.BYTE_ARRAY_OFFSET + byteOffset, + BYTE_ARRAY_OFFSET + byteOffset, numBytes); } static void putBytes(MemoryBuffer buffer, int writerIndex, byte[] bytes, int numBytes) { long address = buffer._unsafeWriterAddress() + writerIndex - buffer.writerIndex(); - UnsafeOps.copyMemory(bytes, UnsafeOps.BYTE_ARRAY_OFFSET, null, address, numBytes); + UNSAFE.copyMemory(bytes, BYTE_ARRAY_OFFSET, null, address, numBytes); } } diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/Serializers.java b/java/fory-core/src/main/java/org/apache/fory/serializer/Serializers.java index 7d808985d2..d8647f19d8 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/Serializers.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/Serializers.java @@ -275,7 +275,55 @@ private static Serializer createSerializer( return createSerializerReflectively(typeResolver, type, serializerClass); } try { + // Public serializers in exported JPMS packages should not require private package opens. + Serializer serializer = + tryCreateSerializer( + typeResolver, type, serializerClass, MethodHandles.publicLookup(), false); + if (serializer != null) { + return serializer; + } + if (!hasSupportedConstructor(serializerClass)) { + throw new IllegalArgumentException( + "Serializer " + + serializerClass.getName() + + " doesn't define a supported constructor for " + + type); + } MethodHandles.Lookup lookup = _JDKAccess._trustedLookup(serializerClass); + return tryCreateSerializer(typeResolver, type, serializerClass, lookup, true); + } catch (Throwable t) { + ExceptionUtils.throwException(t); + throw new IllegalStateException("unreachable"); + } + } + + private static boolean hasSupportedConstructor(Class serializerClass) { + return hasConstructor(serializerClass, TypeResolver.class, Class.class) + || hasConstructor(serializerClass, TypeResolver.class) + || hasConstructor(serializerClass, Config.class, Class.class) + || hasConstructor(serializerClass, Config.class) + || hasConstructor(serializerClass, Class.class) + || hasConstructor(serializerClass); + } + + private static boolean hasConstructor( + Class serializerClass, Class... parameterTypes) { + try { + serializerClass.getDeclaredConstructor(parameterTypes); + return true; + } catch (NoSuchMethodException e) { + return false; + } + } + + private static Serializer tryCreateSerializer( + TypeResolver typeResolver, + Class type, + Class serializerClass, + MethodHandles.Lookup lookup, + boolean checked) + throws Throwable { + try { Config config = typeResolver.getConfig(); try { MethodHandle ctr = lookup.findConstructor(serializerClass, SIG1); @@ -283,6 +331,11 @@ private static Serializer createSerializer( return (Serializer) ctr.invoke(typeResolver, type); } catch (NoSuchMethodException e) { ExceptionUtils.ignore(e); + } catch (IllegalAccessException e) { + if (!checked) { + return null; + } + throw e; } try { MethodHandle ctr = lookup.findConstructor(serializerClass, SIG2); @@ -290,6 +343,11 @@ private static Serializer createSerializer( return (Serializer) ctr.invoke(typeResolver); } catch (NoSuchMethodException e) { ExceptionUtils.ignore(e); + } catch (IllegalAccessException e) { + if (!checked) { + return null; + } + throw e; } try { MethodHandle ctr = lookup.findConstructor(serializerClass, SIG3); @@ -297,6 +355,11 @@ private static Serializer createSerializer( return (Serializer) ctr.invoke(config, type); } catch (NoSuchMethodException e) { ExceptionUtils.ignore(e); + } catch (IllegalAccessException e) { + if (!checked) { + return null; + } + throw e; } try { MethodHandle ctr = lookup.findConstructor(serializerClass, SIG4); @@ -304,6 +367,11 @@ private static Serializer createSerializer( return (Serializer) ctr.invoke(config); } catch (NoSuchMethodException e) { ExceptionUtils.ignore(e); + } catch (IllegalAccessException e) { + if (!checked) { + return null; + } + throw e; } try { MethodHandle ctr = lookup.findConstructor(serializerClass, SIG5); @@ -311,13 +379,38 @@ private static Serializer createSerializer( return (Serializer) ctr.invoke(type); } catch (NoSuchMethodException e) { ExceptionUtils.ignore(e); + } catch (IllegalAccessException e) { + if (!checked) { + return null; + } + throw e; + } + try { + MethodHandle ctr = lookup.findConstructor(serializerClass, SIG6); + CTR_MAP.put(serializerClass, Tuple2.of(SIG6, ctr)); + return (Serializer) ctr.invoke(); + } catch (NoSuchMethodException e) { + ExceptionUtils.ignore(e); + } catch (IllegalAccessException e) { + if (!checked) { + return null; + } + throw e; + } + if (!checked) { + return null; + } + MethodHandle ctr = ReflectionUtils.getCtrHandle(serializerClass, true); + if (ctr == null) { + return null; } - MethodHandle ctr = ReflectionUtils.getCtrHandle(serializerClass); CTR_MAP.put(serializerClass, Tuple2.of(SIG6, ctr)); return (Serializer) ctr.invoke(); - } catch (Throwable t) { - ExceptionUtils.throwException(t); - throw new IllegalStateException("unreachable"); + } catch (IllegalAccessException e) { + if (!checked) { + return null; + } + throw e; } } diff --git a/java/fory-core/src/main/java25/org/apache/fory/memory/LittleEndian.java b/java/fory-core/src/main/java25/org/apache/fory/memory/LittleEndian.java new file mode 100644 index 0000000000..bc7ffd4e9d --- /dev/null +++ b/java/fory-core/src/main/java25/org/apache/fory/memory/LittleEndian.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.memory; + +public class LittleEndian { + public static int putVarUint36Small(byte[] arr, int index, long v) { + if (v >>> 7 == 0) { + arr[index] = (byte) v; + return 1; + } + if (v >>> 14 == 0) { + arr[index++] = (byte) ((v & 0x7F) | 0x80); + arr[index] = (byte) (v >>> 7); + return 2; + } + return bigWriteUint36(arr, index, v); + } + + private static int bigWriteUint36(byte[] arr, int index, long v) { + if (v >>> 21 == 0) { + arr[index++] = (byte) ((v & 0x7F) | 0x80); + arr[index++] = (byte) (v >>> 7 | 0x80); + arr[index] = (byte) (v >>> 14); + return 3; + } + if (v >>> 28 == 0) { + arr[index++] = (byte) ((v & 0x7F) | 0x80); + arr[index++] = (byte) (v >>> 7 | 0x80); + arr[index++] = (byte) (v >>> 14 | 0x80); + arr[index] = (byte) (v >>> 21); + return 4; + } + arr[index++] = (byte) ((v & 0x7F) | 0x80); + arr[index++] = (byte) (v >>> 7 | 0x80); + arr[index++] = (byte) (v >>> 14 | 0x80); + arr[index++] = (byte) (v >>> 21 | 0x80); + arr[index] = (byte) (v >>> 28); + return 5; + } + + public static long getInt64(byte[] o, int index) { + return MemoryOps.getInt64(o, index); + } + + public static void putInt64(byte[] o, int index, long value) { + MemoryOps.putInt64(o, index, value); + } +} diff --git a/java/fory-core/src/main/java25/org/apache/fory/memory/MemoryBuffer.java b/java/fory-core/src/main/java25/org/apache/fory/memory/MemoryBuffer.java index 5232671e14..86827aaeee 100644 --- a/java/fory-core/src/main/java25/org/apache/fory/memory/MemoryBuffer.java +++ b/java/fory-core/src/main/java25/org/apache/fory/memory/MemoryBuffer.java @@ -22,8 +22,6 @@ import static org.apache.fory.util.Preconditions.checkArgument; import static org.apache.fory.util.Preconditions.checkNotNull; -import java.lang.foreign.MemorySegment; -import java.lang.foreign.ValueLayout; import java.lang.invoke.MethodHandles; import java.lang.invoke.VarHandle; import java.nio.ByteBuffer; @@ -33,7 +31,6 @@ import org.apache.fory.io.AbstractStreamReader; import org.apache.fory.io.ForyStreamReader; import org.apache.fory.platform.AndroidSupport; -import org.apache.fory.platform.UnsafeOps; /** * A class for operations on memory managed by Fory. The buffer may be backed by heap memory (byte @@ -69,8 +66,16 @@ public final class MemoryBuffer { public static final int BUFFER_GROW_STEP_THRESHOLD = 100 * 1024 * 1024; private static final boolean LITTLE_ENDIAN = NativeByteOrder.IS_LITTLE_ENDIAN; - private static final boolean UNALIGNED = !AndroidSupport.IS_ANDROID && UnsafeOps.unaligned(); + private static final boolean UNALIGNED = true; private static final ByteOrder NATIVE_ORDER = ByteOrder.nativeOrder(); + private static final int BOOLEAN_ARRAY_OFFSET = 0; + private static final int BYTE_ARRAY_OFFSET = 0; + private static final int CHAR_ARRAY_OFFSET = 0; + private static final int SHORT_ARRAY_OFFSET = 0; + private static final int INT_ARRAY_OFFSET = 0; + private static final int LONG_ARRAY_OFFSET = 0; + private static final int FLOAT_ARRAY_OFFSET = 0; + private static final int DOUBLE_ARRAY_OFFSET = 0; private static final VarHandle BYTE_ARRAY_CHAR = MethodHandles.byteArrayViewVarHandle(char[].class, NATIVE_ORDER); private static final VarHandle BYTE_ARRAY_SHORT = @@ -87,14 +92,6 @@ public final class MemoryBuffer { MethodHandles.byteBufferViewVarHandle(int[].class, NATIVE_ORDER); private static final VarHandle BYTE_BUFFER_LONG = MethodHandles.byteBufferViewVarHandle(long[].class, NATIVE_ORDER); - private static final ValueLayout.OfChar NATIVE_CHAR = - ValueLayout.JAVA_CHAR_UNALIGNED.withOrder(NATIVE_ORDER); - private static final ValueLayout.OfShort NATIVE_SHORT = - ValueLayout.JAVA_SHORT_UNALIGNED.withOrder(NATIVE_ORDER); - private static final ValueLayout.OfInt NATIVE_INT = - ValueLayout.JAVA_INT_UNALIGNED.withOrder(NATIVE_ORDER); - private static final ValueLayout.OfLong NATIVE_LONG = - ValueLayout.JAVA_LONG_UNALIGNED.withOrder(NATIVE_ORDER); // Global allocator instance that can be customized private static volatile MemoryAllocator globalAllocator = new DefaultMemoryAllocator(); @@ -111,7 +108,6 @@ public final class MemoryBuffer { // the memory will not be released. ByteBuffer offHeapBuffer; ByteBuffer nativeOffHeapBuffer; - MemorySegment offHeapSegment; // The readable/writeable range is [address, addressLimit). // If the data in on the heap, this is the relative offset to the `heapMemory` byte array. // If the data is off the heap, this is the logical byte index into `offHeapBuffer`. @@ -124,7 +120,6 @@ public final class MemoryBuffer { int readerIndex; int writerIndex; final ForyStreamReader streamReader; - private final MemoryAccess memoryAccess = new MemoryAccess(this); // Android branches in this class are intentional method-boundary exits. // Do not delete them or fold them into the JVM Unsafe path: each branch must make exactly one @@ -205,10 +200,12 @@ private void initOffHeapBuffer(long offHeapAddress, int size, ByteBuffer offHeap checkNotNull(offHeapBuffer, "JDK25 MemoryBuffer requires a ByteBuffer owner for off-heap data"); checkArgument(offHeapBuffer.isDirect(), "Only direct ByteBuffers can back off-heap MemoryBuffer"); this.offHeapBuffer = offHeapBuffer; - this.nativeOffHeapBuffer = offHeapBuffer.duplicate().order(NATIVE_ORDER); - ByteBuffer segmentBuffer = offHeapBuffer.duplicate(); - ByteBufferUtil.position(segmentBuffer, 0); - this.offHeapSegment = MemorySegment.ofBuffer(segmentBuffer); + ByteBuffer nativeBuffer = offHeapBuffer.duplicate().order(NATIVE_ORDER); + // Stream readers can expand the owner buffer limit after this duplicate is created. Keep the + // absolute-access view capacity-wide so JDK25 public ByteBuffer checks match the logical buffer + // size tracked by MemoryBuffer. + nativeBuffer.clear(); + this.nativeOffHeapBuffer = nativeBuffer; this.heapMemory = null; this.address = offHeapAddress; this.addressLimit = this.address + size; @@ -277,9 +274,7 @@ public void initHeapBuffer(byte[] buffer, int offset, int length) { this.heapOffset = offset; this.offHeapBuffer = null; this.nativeOffHeapBuffer = null; - this.offHeapSegment = null; - // Versioned UnsafeOps array base offsets are zero on JDK25; address is a logical byte index. - final long startPos = UnsafeOps.BYTE_ARRAY_OFFSET + offset; + final long startPos = BYTE_ARRAY_OFFSET + offset; this.address = startPos; this.size = length; this.addressLimit = startPos + length; @@ -371,6 +366,240 @@ private void storeLong(long pos, long value) { } } + private void readBytesToArray(long srcOffset, byte[] target, int targetOffset, int numBytes) { + byte[] heap = heapMemory; + if (heap != null) { + System.arraycopy(heap, toIntIndex(srcOffset), target, targetOffset, numBytes); + } else { + nativeOffHeapBuffer.get(toIntIndex(srcOffset), target, targetOffset, numBytes); + } + } + + private void writeBytesFromArray(long targetOffset, byte[] source, int sourceOffset, int numBytes) { + byte[] heap = heapMemory; + if (heap != null) { + System.arraycopy(source, sourceOffset, heap, toIntIndex(targetOffset), numBytes); + } else { + nativeOffHeapBuffer.put(toIntIndex(targetOffset), source, sourceOffset, numBytes); + } + } + + private void readBooleansToArray( + long srcOffset, boolean[] target, int targetOffset, int numBytes) { + for (int i = 0; i < numBytes; i++) { + target[targetOffset + i] = loadByte(srcOffset + i) != 0; + } + } + + private void writeBooleansFromArray( + long targetOffset, boolean[] source, int sourceOffset, int numBytes) { + for (int i = 0; i < numBytes; i++) { + storeByte(targetOffset + i, source[sourceOffset + i] ? (byte) 1 : (byte) 0); + } + } + + private void readCharsToArray(long srcOffset, char[] target, int targetOffset, int numBytes) { + int elements = numBytes >>> 1; + int pos = toIntIndex(srcOffset); + byte[] heap = heapMemory; + if (heap != null) { + for (int i = 0; i < elements; i++, pos += 2) { + target[targetOffset + i] = (char) BYTE_ARRAY_CHAR.get(heap, pos); + } + } else { + ByteBuffer direct = nativeOffHeapBuffer; + for (int i = 0; i < elements; i++, pos += 2) { + target[targetOffset + i] = (char) BYTE_BUFFER_CHAR.get(direct, pos); + } + } + } + + private void writeCharsFromArray(long targetOffset, char[] source, int sourceOffset, int numBytes) { + int elements = numBytes >>> 1; + int pos = toIntIndex(targetOffset); + byte[] heap = heapMemory; + if (heap != null) { + for (int i = 0; i < elements; i++, pos += 2) { + BYTE_ARRAY_CHAR.set(heap, pos, source[sourceOffset + i]); + } + } else { + ByteBuffer direct = nativeOffHeapBuffer; + for (int i = 0; i < elements; i++, pos += 2) { + BYTE_BUFFER_CHAR.set(direct, pos, source[sourceOffset + i]); + } + } + } + + private void readShortsToArray(long srcOffset, short[] target, int targetOffset, int numBytes) { + int elements = numBytes >>> 1; + int pos = toIntIndex(srcOffset); + byte[] heap = heapMemory; + if (heap != null) { + for (int i = 0; i < elements; i++, pos += 2) { + target[targetOffset + i] = (short) BYTE_ARRAY_SHORT.get(heap, pos); + } + } else { + ByteBuffer direct = nativeOffHeapBuffer; + for (int i = 0; i < elements; i++, pos += 2) { + target[targetOffset + i] = (short) BYTE_BUFFER_SHORT.get(direct, pos); + } + } + } + + private void writeShortsFromArray( + long targetOffset, short[] source, int sourceOffset, int numBytes) { + int elements = numBytes >>> 1; + int pos = toIntIndex(targetOffset); + byte[] heap = heapMemory; + if (heap != null) { + for (int i = 0; i < elements; i++, pos += 2) { + BYTE_ARRAY_SHORT.set(heap, pos, source[sourceOffset + i]); + } + } else { + ByteBuffer direct = nativeOffHeapBuffer; + for (int i = 0; i < elements; i++, pos += 2) { + BYTE_BUFFER_SHORT.set(direct, pos, source[sourceOffset + i]); + } + } + } + + private void readIntsToArray(long srcOffset, int[] target, int targetOffset, int numBytes) { + int elements = numBytes >>> 2; + int pos = toIntIndex(srcOffset); + byte[] heap = heapMemory; + if (heap != null) { + for (int i = 0; i < elements; i++, pos += 4) { + target[targetOffset + i] = (int) BYTE_ARRAY_INT.get(heap, pos); + } + } else { + ByteBuffer direct = nativeOffHeapBuffer; + for (int i = 0; i < elements; i++, pos += 4) { + target[targetOffset + i] = (int) BYTE_BUFFER_INT.get(direct, pos); + } + } + } + + private void writeIntsFromArray(long targetOffset, int[] source, int sourceOffset, int numBytes) { + int elements = numBytes >>> 2; + int pos = toIntIndex(targetOffset); + byte[] heap = heapMemory; + if (heap != null) { + for (int i = 0; i < elements; i++, pos += 4) { + BYTE_ARRAY_INT.set(heap, pos, source[sourceOffset + i]); + } + } else { + ByteBuffer direct = nativeOffHeapBuffer; + for (int i = 0; i < elements; i++, pos += 4) { + BYTE_BUFFER_INT.set(direct, pos, source[sourceOffset + i]); + } + } + } + + private void readLongsToArray(long srcOffset, long[] target, int targetOffset, int numBytes) { + int elements = numBytes >>> 3; + int pos = toIntIndex(srcOffset); + byte[] heap = heapMemory; + if (heap != null) { + for (int i = 0; i < elements; i++, pos += 8) { + target[targetOffset + i] = (long) BYTE_ARRAY_LONG.get(heap, pos); + } + } else { + ByteBuffer direct = nativeOffHeapBuffer; + for (int i = 0; i < elements; i++, pos += 8) { + target[targetOffset + i] = (long) BYTE_BUFFER_LONG.get(direct, pos); + } + } + } + + private void writeLongsFromArray(long targetOffset, long[] source, int sourceOffset, int numBytes) { + int elements = numBytes >>> 3; + int pos = toIntIndex(targetOffset); + byte[] heap = heapMemory; + if (heap != null) { + for (int i = 0; i < elements; i++, pos += 8) { + BYTE_ARRAY_LONG.set(heap, pos, source[sourceOffset + i]); + } + } else { + ByteBuffer direct = nativeOffHeapBuffer; + for (int i = 0; i < elements; i++, pos += 8) { + BYTE_BUFFER_LONG.set(direct, pos, source[sourceOffset + i]); + } + } + } + + private void readFloatsToArray(long srcOffset, float[] target, int targetOffset, int numBytes) { + int elements = numBytes >>> 2; + int pos = toIntIndex(srcOffset); + byte[] heap = heapMemory; + if (heap != null) { + for (int i = 0; i < elements; i++, pos += 4) { + target[targetOffset + i] = Float.intBitsToFloat((int) BYTE_ARRAY_INT.get(heap, pos)); + } + } else { + ByteBuffer direct = nativeOffHeapBuffer; + for (int i = 0; i < elements; i++, pos += 4) { + target[targetOffset + i] = Float.intBitsToFloat((int) BYTE_BUFFER_INT.get(direct, pos)); + } + } + } + + private void writeFloatsFromArray( + long targetOffset, float[] source, int sourceOffset, int numBytes) { + int elements = numBytes >>> 2; + int pos = toIntIndex(targetOffset); + byte[] heap = heapMemory; + if (heap != null) { + for (int i = 0; i < elements; i++, pos += 4) { + BYTE_ARRAY_INT.set(heap, pos, Float.floatToRawIntBits(source[sourceOffset + i])); + } + } else { + ByteBuffer direct = nativeOffHeapBuffer; + for (int i = 0; i < elements; i++, pos += 4) { + BYTE_BUFFER_INT.set(direct, pos, Float.floatToRawIntBits(source[sourceOffset + i])); + } + } + } + + private void readDoublesToArray(long srcOffset, double[] target, int targetOffset, int numBytes) { + int elements = numBytes >>> 3; + int pos = toIntIndex(srcOffset); + byte[] heap = heapMemory; + if (heap != null) { + for (int i = 0; i < elements; i++, pos += 8) { + target[targetOffset + i] = Double.longBitsToDouble((long) BYTE_ARRAY_LONG.get(heap, pos)); + } + } else { + ByteBuffer direct = nativeOffHeapBuffer; + for (int i = 0; i < elements; i++, pos += 8) { + target[targetOffset + i] = Double.longBitsToDouble((long) BYTE_BUFFER_LONG.get(direct, pos)); + } + } + } + + private void writeDoublesFromArray( + long targetOffset, double[] source, int sourceOffset, int numBytes) { + int elements = numBytes >>> 3; + int pos = toIntIndex(targetOffset); + byte[] heap = heapMemory; + if (heap != null) { + for (int i = 0; i < elements; i++, pos += 8) { + BYTE_ARRAY_LONG.set(heap, pos, Double.doubleToRawLongBits(source[sourceOffset + i])); + } + } else { + ByteBuffer direct = nativeOffHeapBuffer; + for (int i = 0; i < elements; i++, pos += 8) { + BYTE_BUFFER_LONG.set(direct, pos, Double.doubleToRawLongBits(source[sourceOffset + i])); + } + } + } + + private static int toIntIndex(long offset) { + if (offset < 0 || offset > Integer.MAX_VALUE) { + throw new IndexOutOfBoundsException("offset out of int range: " + offset); + } + return (int) offset; + } + private static int putVarUInt32Heap(byte[] heap, int index, int value) { if (value >>> 7 == 0) { heap[index] = (byte) value; @@ -500,7 +729,7 @@ public void get(int index, byte[] dst, int offset, int length) { < 0) { throwOOBException(); } - memoryAccess.copyMemory(null, pos, dst, UnsafeOps.BYTE_ARRAY_OFFSET + offset, length); + readBytesToArray(pos, dst, BYTE_ARRAY_OFFSET + offset, length); } } @@ -587,8 +816,7 @@ public void put(int index, byte[] src, int offset, int length) { < 0) { throwOOBException(); } - final long arrayAddress = UnsafeOps.BYTE_ARRAY_OFFSET + offset; - memoryAccess.copyMemory(src, arrayAddress, null, pos, length); + writeBytesFromArray(pos, src, BYTE_ARRAY_OFFSET + offset, length); } } @@ -1873,12 +2101,7 @@ public void writeBooleans(boolean[] values, int offset, int numElements) { final int writerIdx = writerIndex; final int newIdx = writerIdx + numElements; ensure(newIdx); - memoryAccess.copyMemory( - values, - UnsafeOps.BOOLEAN_ARRAY_OFFSET + offset, - heapMemory, - address + writerIdx, - numElements); + writeBooleansFromArray(address + writerIdx, values, BOOLEAN_ARRAY_OFFSET + offset, numElements); writerIndex = newIdx; } } @@ -1905,12 +2128,7 @@ public void writeChars(char[] values, int offset, int numElements) { final int writerIdx = writerIndex; final int newIdx = writerIdx + numBytes; ensure(newIdx); - memoryAccess.copyMemory( - values, - UnsafeOps.CHAR_ARRAY_OFFSET + ((long) offset << 1), - heapMemory, - address + writerIdx, - numBytes); + writeCharsFromArray(address + writerIdx, values, CHAR_ARRAY_OFFSET + offset, numBytes); writerIndex = newIdx; } } @@ -1937,12 +2155,7 @@ public void writeShorts(short[] values, int offset, int numElements) { final int writerIdx = writerIndex; final int newIdx = writerIdx + numBytes; ensure(newIdx); - memoryAccess.copyMemory( - values, - UnsafeOps.SHORT_ARRAY_OFFSET + ((long) offset << 1), - heapMemory, - address + writerIdx, - numBytes); + writeShortsFromArray(address + writerIdx, values, SHORT_ARRAY_OFFSET + offset, numBytes); writerIndex = newIdx; } } @@ -1969,12 +2182,7 @@ public void writeInts(int[] values, int offset, int numElements) { final int writerIdx = writerIndex; final int newIdx = writerIdx + numBytes; ensure(newIdx); - memoryAccess.copyMemory( - values, - UnsafeOps.INT_ARRAY_OFFSET + ((long) offset << 2), - heapMemory, - address + writerIdx, - numBytes); + writeIntsFromArray(address + writerIdx, values, INT_ARRAY_OFFSET + offset, numBytes); writerIndex = newIdx; } } @@ -2001,12 +2209,7 @@ public void writeLongs(long[] values, int offset, int numElements) { final int writerIdx = writerIndex; final int newIdx = writerIdx + numBytes; ensure(newIdx); - memoryAccess.copyMemory( - values, - UnsafeOps.LONG_ARRAY_OFFSET + ((long) offset << 3), - heapMemory, - address + writerIdx, - numBytes); + writeLongsFromArray(address + writerIdx, values, LONG_ARRAY_OFFSET + offset, numBytes); writerIndex = newIdx; } } @@ -2033,12 +2236,7 @@ public void writeFloats(float[] values, int offset, int numElements) { final int writerIdx = writerIndex; final int newIdx = writerIdx + numBytes; ensure(newIdx); - memoryAccess.copyMemory( - values, - UnsafeOps.FLOAT_ARRAY_OFFSET + ((long) offset << 2), - heapMemory, - address + writerIdx, - numBytes); + writeFloatsFromArray(address + writerIdx, values, FLOAT_ARRAY_OFFSET + offset, numBytes); writerIndex = newIdx; } } @@ -2065,12 +2263,7 @@ public void writeDoubles(double[] values, int offset, int numElements) { final int writerIdx = writerIndex; final int newIdx = writerIdx + numBytes; ensure(newIdx); - memoryAccess.copyMemory( - values, - UnsafeOps.DOUBLE_ARRAY_OFFSET + ((long) offset << 3), - heapMemory, - address + writerIdx, - numBytes); + writeDoublesFromArray(address + writerIdx, values, DOUBLE_ARRAY_OFFSET + offset, numBytes); writerIndex = newIdx; } } @@ -3244,13 +3437,7 @@ public byte[] readBytes(int length) { streamReader.readTo(bytes, 0, length); return bytes; } - byte[] heapMemory = this.heapMemory; - if (heapMemory != null) { - // System.arraycopy faster for some jdk than Unsafe. - System.arraycopy(heapMemory, heapOffset + readerIdx, bytes, 0, length); - } else { - memoryAccess.copyMemory(null, address + readerIdx, bytes, UnsafeOps.BYTE_ARRAY_OFFSET, length); - } + readBytesToArray(address + readerIdx, bytes, BYTE_ARRAY_OFFSET, length); readerIndex = readerIdx + length; return bytes; } @@ -3305,7 +3492,7 @@ private long slowReadBytesAsInt64(int remaining, int len) { } else { long start = address + readerIdx; for (int i = 0; i < len; i++) { - result |= ((long) memoryAccess.getByte(null, start + i) & 0xff) << (i * 8); + result |= ((long) loadByte(start + i) & 0xff) << (i * 8); } } return result; @@ -3415,13 +3602,7 @@ public byte[] readBytesAndSize() { streamReader.readTo(arr, 0, numBytes); return arr; } - byte[] heapMemory = this.heapMemory; - if (heapMemory != null) { - System.arraycopy(heapMemory, heapOffset + readerIdx, arr, 0, numBytes); - } else { - memoryAccess.copyMemory( - null, address + readerIdx, arr, UnsafeOps.BYTE_ARRAY_OFFSET, numBytes); - } + readBytesToArray(address + readerIdx, arr, BYTE_ARRAY_OFFSET, numBytes); readerIndex = readerIdx + numBytes; return arr; } @@ -3440,12 +3621,7 @@ public void readByteArrayPayload(byte[] values, int numBytes) { streamReader.readTo(values, 0, numBytes); return; } - byte[] heapMemory = this.heapMemory; - if (heapMemory != null) { - System.arraycopy(heapMemory, heapOffset + readerIdx, values, 0, numBytes); - } else { - memoryAccess.copyMemory(null, address + readerIdx, values, UnsafeOps.BYTE_ARRAY_OFFSET, numBytes); - } + readBytesToArray(address + readerIdx, values, BYTE_ARRAY_OFFSET, numBytes); readerIndex = readerIdx + numBytes; } } @@ -3464,8 +3640,7 @@ public void readBooleanArrayPayload(boolean[] values, int numBytes) { streamReader.readBooleans(values, 0, numBytes); return; } - memoryAccess.copyMemory( - heapMemory, address + readerIdx, values, UnsafeOps.BOOLEAN_ARRAY_OFFSET, numBytes); + readBooleansToArray(address + readerIdx, values, BOOLEAN_ARRAY_OFFSET, numBytes); readerIndex = readerIdx + numBytes; } } @@ -3484,8 +3659,7 @@ public void readCharArrayPayload(char[] values, int numBytes) { streamReader.readChars(values, 0, numBytes >>> 1); return; } - memoryAccess.copyMemory( - heapMemory, address + readerIdx, values, UnsafeOps.CHAR_ARRAY_OFFSET, numBytes); + readCharsToArray(address + readerIdx, values, CHAR_ARRAY_OFFSET, numBytes); readerIndex = readerIdx + numBytes; } } @@ -3504,8 +3678,7 @@ public void readInt16ArrayPayload(short[] values, int numBytes) { streamReader.readShorts(values, 0, numBytes >>> 1); return; } - memoryAccess.copyMemory( - heapMemory, address + readerIdx, values, UnsafeOps.SHORT_ARRAY_OFFSET, numBytes); + readShortsToArray(address + readerIdx, values, SHORT_ARRAY_OFFSET, numBytes); readerIndex = readerIdx + numBytes; } } @@ -3524,8 +3697,7 @@ public void readInt32ArrayPayload(int[] values, int numBytes) { streamReader.readInts(values, 0, numBytes >>> 2); return; } - memoryAccess.copyMemory( - heapMemory, address + readerIdx, values, UnsafeOps.INT_ARRAY_OFFSET, numBytes); + readIntsToArray(address + readerIdx, values, INT_ARRAY_OFFSET, numBytes); readerIndex = readerIdx + numBytes; } } @@ -3544,8 +3716,7 @@ public void readInt64ArrayPayload(long[] values, int numBytes) { streamReader.readLongs(values, 0, numBytes >>> 3); return; } - memoryAccess.copyMemory( - heapMemory, address + readerIdx, values, UnsafeOps.LONG_ARRAY_OFFSET, numBytes); + readLongsToArray(address + readerIdx, values, LONG_ARRAY_OFFSET, numBytes); readerIndex = readerIdx + numBytes; } } @@ -3564,8 +3735,7 @@ public void readFloat32ArrayPayload(float[] values, int numBytes) { streamReader.readFloats(values, 0, numBytes >>> 2); return; } - memoryAccess.copyMemory( - heapMemory, address + readerIdx, values, UnsafeOps.FLOAT_ARRAY_OFFSET, numBytes); + readFloatsToArray(address + readerIdx, values, FLOAT_ARRAY_OFFSET, numBytes); readerIndex = readerIdx + numBytes; } } @@ -3584,8 +3754,7 @@ public void readFloat64ArrayPayload(double[] values, int numBytes) { streamReader.readDoubles(values, 0, numBytes >>> 3); return; } - memoryAccess.copyMemory( - heapMemory, address + readerIdx, values, UnsafeOps.DOUBLE_ARRAY_OFFSET, numBytes); + readDoublesToArray(address + readerIdx, values, DOUBLE_ARRAY_OFFSET, numBytes); readerIndex = readerIdx + numBytes; } } @@ -3603,12 +3772,7 @@ public void readBooleans(boolean[] values, int offset, int numElements) { return; } int readerIdx = readerIndex; - memoryAccess.copyMemory( - heapMemory, - address + readerIdx, - values, - UnsafeOps.BOOLEAN_ARRAY_OFFSET + offset, - numElements); + readBooleansToArray(address + readerIdx, values, BOOLEAN_ARRAY_OFFSET + offset, numElements); readerIndex = readerIdx + numElements; } } @@ -3631,12 +3795,7 @@ public void readChars(char[] chars, int offset, int numElements) { return; } int readerIdx = readerIndex; - memoryAccess.copyMemory( - heapMemory, - address + readerIdx, - chars, - UnsafeOps.CHAR_ARRAY_OFFSET + ((long) offset << 1), - numBytes); + readCharsToArray(address + readerIdx, chars, CHAR_ARRAY_OFFSET + offset, numBytes); readerIndex = readerIdx + numBytes; } } @@ -3664,12 +3823,7 @@ public void readShorts(short[] values, int offset, int numElements) { return; } int readerIdx = readerIndex; - memoryAccess.copyMemory( - heapMemory, - address + readerIdx, - values, - UnsafeOps.SHORT_ARRAY_OFFSET + ((long) offset << 1), - numBytes); + readShortsToArray(address + readerIdx, values, SHORT_ARRAY_OFFSET + offset, numBytes); readerIndex = readerIdx + numBytes; } } @@ -3688,12 +3842,7 @@ public void readInts(int[] values, int offset, int numElements) { return; } int readerIdx = readerIndex; - memoryAccess.copyMemory( - heapMemory, - address + readerIdx, - values, - UnsafeOps.INT_ARRAY_OFFSET + ((long) offset << 2), - numBytes); + readIntsToArray(address + readerIdx, values, INT_ARRAY_OFFSET + offset, numBytes); readerIndex = readerIdx + numBytes; } } @@ -3712,12 +3861,7 @@ public void readLongs(long[] values, int offset, int numElements) { return; } int readerIdx = readerIndex; - memoryAccess.copyMemory( - heapMemory, - address + readerIdx, - values, - UnsafeOps.LONG_ARRAY_OFFSET + ((long) offset << 3), - numBytes); + readLongsToArray(address + readerIdx, values, LONG_ARRAY_OFFSET + offset, numBytes); readerIndex = readerIdx + numBytes; } } @@ -3736,12 +3880,7 @@ public void readFloats(float[] values, int offset, int numElements) { return; } int readerIdx = readerIndex; - memoryAccess.copyMemory( - heapMemory, - address + readerIdx, - values, - UnsafeOps.FLOAT_ARRAY_OFFSET + ((long) offset << 2), - numBytes); + readFloatsToArray(address + readerIdx, values, FLOAT_ARRAY_OFFSET + offset, numBytes); readerIndex = readerIdx + numBytes; } } @@ -3760,12 +3899,7 @@ public void readDoubles(double[] values, int offset, int numElements) { return; } int readerIdx = readerIndex; - memoryAccess.copyMemory( - heapMemory, - address + readerIdx, - values, - UnsafeOps.DOUBLE_ARRAY_OFFSET + ((long) offset << 3), - numBytes); + readDoublesToArray(address + readerIdx, values, DOUBLE_ARRAY_OFFSET + offset, numBytes); readerIndex = readerIdx + numBytes; } } @@ -3805,7 +3939,7 @@ public void copyTo(int offset, MemoryBuffer target, int targetOffset, int numByt && thisPointer <= this.addressLimit - numBytes && otherPointer <= target.addressLimit - numBytes) { if (thisHeapRef != null && otherHeapRef != null) { - memoryAccess.copyMemory(thisHeapRef, thisPointer, otherHeapRef, otherPointer, numBytes); + System.arraycopy(thisHeapRef, toIntIndex(thisPointer), otherHeapRef, toIntIndex(otherPointer), numBytes); } else if (sameBufferOverlap(target, offset, targetOffset, numBytes)) { byte[] tmp = new byte[numBytes]; sliceAsByteBuffer(offset, numBytes).get(tmp); @@ -3849,7 +3983,7 @@ public void copyToByteArray(int offset, byte[] target, int targetOffset, int num MemoryOps.copyToByteArray(this, offset, target, targetOffset, numBytes); } else { checkArrayCopy(offset, targetOffset, target.length, numBytes, 0); - memoryAccess.copyMemory(heapMemory, address + offset, target, targetOffset, numBytes); + readBytesToArray(address + offset, target, BYTE_ARRAY_OFFSET + targetOffset, numBytes); } } @@ -3858,7 +3992,7 @@ public void copyToBooleanArray(int offset, boolean[] target, int targetOffset, i MemoryOps.copyToBooleanArray(this, offset, target, targetOffset, numBytes); } else { checkArrayCopy(offset, targetOffset, target.length, numBytes, 0); - memoryAccess.copyMemory(heapMemory, address + offset, target, targetOffset, numBytes); + readBooleansToArray(address + offset, target, BOOLEAN_ARRAY_OFFSET + targetOffset, numBytes); } } @@ -3867,8 +4001,7 @@ public void copyToCharArray(int offset, char[] target, int targetOffset, int num MemoryOps.copyToCharArray(this, offset, target, targetOffset, numBytes); } else { checkArrayCopy(offset, targetOffset, target.length, numBytes, 1); - memoryAccess.copyMemory( - heapMemory, address + offset, target, arrayCopyOffset(targetOffset, 1), numBytes); + readCharsToArray(address + offset, target, CHAR_ARRAY_OFFSET + targetOffset, numBytes); } } @@ -3877,8 +4010,7 @@ public void copyToShortArray(int offset, short[] target, int targetOffset, int n MemoryOps.copyToShortArray(this, offset, target, targetOffset, numBytes); } else { checkArrayCopy(offset, targetOffset, target.length, numBytes, 1); - memoryAccess.copyMemory( - heapMemory, address + offset, target, arrayCopyOffset(targetOffset, 1), numBytes); + readShortsToArray(address + offset, target, SHORT_ARRAY_OFFSET + targetOffset, numBytes); } } @@ -3887,8 +4019,7 @@ public void copyToIntArray(int offset, int[] target, int targetOffset, int numBy MemoryOps.copyToIntArray(this, offset, target, targetOffset, numBytes); } else { checkArrayCopy(offset, targetOffset, target.length, numBytes, 2); - memoryAccess.copyMemory( - heapMemory, address + offset, target, arrayCopyOffset(targetOffset, 2), numBytes); + readIntsToArray(address + offset, target, INT_ARRAY_OFFSET + targetOffset, numBytes); } } @@ -3897,8 +4028,7 @@ public void copyToLongArray(int offset, long[] target, int targetOffset, int num MemoryOps.copyToLongArray(this, offset, target, targetOffset, numBytes); } else { checkArrayCopy(offset, targetOffset, target.length, numBytes, 3); - memoryAccess.copyMemory( - heapMemory, address + offset, target, arrayCopyOffset(targetOffset, 3), numBytes); + readLongsToArray(address + offset, target, LONG_ARRAY_OFFSET + targetOffset, numBytes); } } @@ -3907,8 +4037,7 @@ public void copyToFloatArray(int offset, float[] target, int targetOffset, int n MemoryOps.copyToFloatArray(this, offset, target, targetOffset, numBytes); } else { checkArrayCopy(offset, targetOffset, target.length, numBytes, 2); - memoryAccess.copyMemory( - heapMemory, address + offset, target, arrayCopyOffset(targetOffset, 2), numBytes); + readFloatsToArray(address + offset, target, FLOAT_ARRAY_OFFSET + targetOffset, numBytes); } } @@ -3917,8 +4046,7 @@ public void copyToDoubleArray(int offset, double[] target, int targetOffset, int MemoryOps.copyToDoubleArray(this, offset, target, targetOffset, numBytes); } else { checkArrayCopy(offset, targetOffset, target.length, numBytes, 3); - memoryAccess.copyMemory( - heapMemory, address + offset, target, arrayCopyOffset(targetOffset, 3), numBytes); + readDoublesToArray(address + offset, target, DOUBLE_ARRAY_OFFSET + targetOffset, numBytes); } } @@ -3936,22 +4064,76 @@ private void checkArrayCopy( } } - private static long arrayCopyOffset(int elementOffset, int elementShift) { - return (long) elementOffset << elementShift; + public void copyFromByteArray(int offset, byte[] source, int sourceOffset, int numBytes) { + if (AndroidSupport.IS_ANDROID) { + MemoryOps.copyFromByteArray(this, offset, source, sourceOffset, numBytes); + } else { + checkArrayCopy(offset, sourceOffset, source.length, numBytes, 0); + writeBytesFromArray(address + offset, source, BYTE_ARRAY_OFFSET + sourceOffset, numBytes); + } + } + + public void copyFromBooleanArray(int offset, boolean[] source, int sourceOffset, int numBytes) { + if (AndroidSupport.IS_ANDROID) { + MemoryOps.copyFromBooleanArray(this, offset, source, sourceOffset, numBytes); + } else { + checkArrayCopy(offset, sourceOffset, source.length, numBytes, 0); + writeBooleansFromArray( + address + offset, source, BOOLEAN_ARRAY_OFFSET + sourceOffset, numBytes); + } } - /** - * JVM-only bulk copy method. Copies {@code numBytes} bytes from source unsafe object and pointer. - * Throws on Android before executing unsafe memory access. - */ - public void copyFromUnsafe(long offset, Object source, long sourcePointer, long numBytes) { + public void copyFromCharArray(int offset, char[] source, int sourceOffset, int numBytes) { if (AndroidSupport.IS_ANDROID) { - MemoryOps.throwRawUnsafeMemoryCopyUnsupported(); + MemoryOps.copyFromCharArray(this, offset, source, sourceOffset, numBytes); } else { - checkArgument(source != null, "Raw native-address source copy is unsupported on JDK25"); - final long thisPointer = this.address + offset; - checkArgument(thisPointer + numBytes <= addressLimit); - memoryAccess.copyMemory(source, sourcePointer, heapMemory, thisPointer, numBytes); + checkArrayCopy(offset, sourceOffset, source.length, numBytes, 1); + writeCharsFromArray(address + offset, source, CHAR_ARRAY_OFFSET + sourceOffset, numBytes); + } + } + + public void copyFromShortArray(int offset, short[] source, int sourceOffset, int numBytes) { + if (AndroidSupport.IS_ANDROID) { + MemoryOps.copyFromShortArray(this, offset, source, sourceOffset, numBytes); + } else { + checkArrayCopy(offset, sourceOffset, source.length, numBytes, 1); + writeShortsFromArray(address + offset, source, SHORT_ARRAY_OFFSET + sourceOffset, numBytes); + } + } + + public void copyFromIntArray(int offset, int[] source, int sourceOffset, int numBytes) { + if (AndroidSupport.IS_ANDROID) { + MemoryOps.copyFromIntArray(this, offset, source, sourceOffset, numBytes); + } else { + checkArrayCopy(offset, sourceOffset, source.length, numBytes, 2); + writeIntsFromArray(address + offset, source, INT_ARRAY_OFFSET + sourceOffset, numBytes); + } + } + + public void copyFromLongArray(int offset, long[] source, int sourceOffset, int numBytes) { + if (AndroidSupport.IS_ANDROID) { + MemoryOps.copyFromLongArray(this, offset, source, sourceOffset, numBytes); + } else { + checkArrayCopy(offset, sourceOffset, source.length, numBytes, 3); + writeLongsFromArray(address + offset, source, LONG_ARRAY_OFFSET + sourceOffset, numBytes); + } + } + + public void copyFromFloatArray(int offset, float[] source, int sourceOffset, int numBytes) { + if (AndroidSupport.IS_ANDROID) { + MemoryOps.copyFromFloatArray(this, offset, source, sourceOffset, numBytes); + } else { + checkArrayCopy(offset, sourceOffset, source.length, numBytes, 2); + writeFloatsFromArray(address + offset, source, FLOAT_ARRAY_OFFSET + sourceOffset, numBytes); + } + } + + public void copyFromDoubleArray(int offset, double[] source, int sourceOffset, int numBytes) { + if (AndroidSupport.IS_ANDROID) { + MemoryOps.copyFromDoubleArray(this, offset, source, sourceOffset, numBytes); + } else { + checkArrayCopy(offset, sourceOffset, source.length, numBytes, 3); + writeDoublesFromArray(address + offset, source, DOUBLE_ARRAY_OFFSET + sourceOffset, numBytes); } } @@ -4008,6 +4190,7 @@ public ByteBuffer sliceAsByteBuffer(int offset, int length) { if (offHeapBuffer != null) { ByteBuffer duplicate = offHeapBuffer.duplicate(); int start = (int) address; + duplicate.clear(); ByteBufferUtil.position(duplicate, start + offset); duplicate.limit(start + offset + length); return duplicate.slice(); @@ -4046,7 +4229,7 @@ public boolean equalTo(MemoryBuffer buf2, int offset1, int offset2, int len) { final long pos2 = buf2.address + offset2; checkArgument(pos1 < addressLimit); checkArgument(pos2 < buf2.addressLimit); - return unsafeEqualTo(memoryAccess, heapMemory, pos1, buf2.memoryAccess, buf2.heapMemory, pos2, len); + return unsafeEqualTo(this, heapMemory, pos1, buf2, buf2.heapMemory, pos2, len); } /** @@ -4070,23 +4253,21 @@ public boolean equalTo(byte[] bytes, int bytesOffset, int offset, int len) { return MemoryOps.equalTo(this, bytes, bytesOffset, offset, len); } final long pos = address + offset; - return unsafeEqualTo( - memoryAccess, heapMemory, pos, memoryAccess, bytes, UnsafeOps.BYTE_ARRAY_OFFSET + bytesOffset, len); + return unsafeEqualTo(this, heapMemory, pos, this, bytes, BYTE_ARRAY_OFFSET + bytesOffset, len); } private static boolean unsafeEqualTo( - MemoryAccess leftAccess, + MemoryBuffer left, Object leftBase, long leftOffset, - MemoryAccess rightAccess, + MemoryBuffer right, Object rightBase, long rightOffset, int length) { int i = 0; if ((leftOffset % 8) == (rightOffset % 8)) { while ((leftOffset + i) % 8 != 0 && i < length) { - if (leftAccess.getByte(leftBase, leftOffset + i) - != rightAccess.getByte(rightBase, rightOffset + i)) { + if (left.rawByte(leftBase, leftOffset + i) != right.rawByte(rightBase, rightOffset + i)) { return false; } i += 1; @@ -4094,16 +4275,14 @@ private static boolean unsafeEqualTo( } if (UNALIGNED || (((leftOffset + i) % 8 == 0) && ((rightOffset + i) % 8 == 0))) { while (i <= length - 8) { - if (leftAccess.getLong(leftBase, leftOffset + i) - != rightAccess.getLong(rightBase, rightOffset + i)) { + if (left.rawLong(leftBase, leftOffset + i) != right.rawLong(rightBase, rightOffset + i)) { return false; } i += 8; } } while (i < length) { - if (leftAccess.getByte(leftBase, leftOffset + i) - != rightAccess.getByte(rightBase, rightOffset + i)) { + if (left.rawByte(leftBase, leftOffset + i) != right.rawByte(rightBase, rightOffset + i)) { return false; } i += 1; @@ -4111,361 +4290,18 @@ private static boolean unsafeEqualTo( return true; } - private static final class MemoryAccess { - private final MemoryBuffer buffer; - - private MemoryAccess(MemoryBuffer buffer) { - this.buffer = buffer; - } - - private byte getByte(Object base, long offset) { - if (base != null) { - return UnsafeOps.getByte(base, offset); - } - return directSegment().get(ValueLayout.JAVA_BYTE, offset); - } - - private void putByte(Object base, long offset, byte value) { - if (base != null) { - UnsafeOps.putByte(base, offset, value); - return; - } - directSegment().set(ValueLayout.JAVA_BYTE, offset, value); - } - - private char getChar(Object base, long offset) { - if (base != null) { - return UnsafeOps.getChar(base, offset); - } - return directSegment().get(NATIVE_CHAR, offset); - } - - private void putChar(Object base, long offset, char value) { - if (base != null) { - UnsafeOps.putChar(base, offset, value); - return; - } - directSegment().set(NATIVE_CHAR, offset, value); - } - - private short getShort(Object base, long offset) { - if (base != null) { - return UnsafeOps.getShort(base, offset); - } - return directSegment().get(NATIVE_SHORT, offset); - } - - private void putShort(Object base, long offset, short value) { - if (base != null) { - UnsafeOps.putShort(base, offset, value); - return; - } - directSegment().set(NATIVE_SHORT, offset, value); - } - - private int getInt(Object base, long offset) { - if (base != null) { - return UnsafeOps.getInt(base, offset); - } - return directSegment().get(NATIVE_INT, offset); - } - - private void putInt(Object base, long offset, int value) { - if (base != null) { - UnsafeOps.putInt(base, offset, value); - return; - } - directSegment().set(NATIVE_INT, offset, value); - } - - private long getLong(Object base, long offset) { - if (base != null) { - return UnsafeOps.getLong(base, offset); - } - return directSegment().get(NATIVE_LONG, offset); - } - - private void putLong(Object base, long offset, long value) { - if (base != null) { - UnsafeOps.putLong(base, offset, value); - return; - } - directSegment().set(NATIVE_LONG, offset, value); - } - - private void copyMemory( - Object src, long srcOffset, Object dst, long dstOffset, long length) { - int len = toIntLength(length); - if (len == 0) { - return; - } - if (src != null && dst != null) { - if (dst instanceof byte[] && copyArrayToBytes(src, srcOffset, (byte[]) dst, toIntIndex(dstOffset), len)) { - return; - } - if (src instanceof byte[] && copyBytesToArray((byte[]) src, toIntIndex(srcOffset), dst, dstOffset, len)) { - return; - } - UnsafeOps.copyMemory(src, srcOffset, dst, dstOffset, len); - } else if (src == null && dst == null) { - copyDirect(srcOffset, dstOffset, len); - } else if (src == null) { - if (dst instanceof byte[]) { - readDirect(srcOffset, (byte[]) dst, toIntIndex(dstOffset), len); - } else if (readArray(srcOffset, dst, dstOffset, len)) { - return; - } else { - for (int i = 0; i < len; i++) { - UnsafeOps.putByte(dst, dstOffset + i, getByte(null, srcOffset + i)); - } - } - } else if (src instanceof byte[]) { - writeDirect(dstOffset, (byte[]) src, toIntIndex(srcOffset), len); - } else if (writeArray(src, srcOffset, dstOffset, len)) { - return; - } else { - for (int i = 0; i < len; i++) { - putByte(null, dstOffset + i, UnsafeOps.getByte(src, srcOffset + i)); - } - } - } - - private boolean copyArrayToBytes( - Object src, long srcOffset, byte[] dst, int dstOffset, int len) { - if (src instanceof boolean[]) { - boolean[] array = (boolean[]) src; - int srcIndex = toIntIndex(srcOffset); - for (int i = 0; i < len; i++) { - dst[dstOffset + i] = array[srcIndex + i] ? (byte) 1 : (byte) 0; - } - return true; - } else if (src instanceof char[] && aligned(srcOffset, len, Character.BYTES)) { - heapBytes(dst, dstOffset, len) - .asCharBuffer() - .put((char[]) src, toIntIndex(srcOffset / Character.BYTES), len / Character.BYTES); - return true; - } else if (src instanceof short[] && aligned(srcOffset, len, Short.BYTES)) { - heapBytes(dst, dstOffset, len) - .asShortBuffer() - .put((short[]) src, toIntIndex(srcOffset / Short.BYTES), len / Short.BYTES); - return true; - } else if (src instanceof int[] && aligned(srcOffset, len, Integer.BYTES)) { - heapBytes(dst, dstOffset, len) - .asIntBuffer() - .put((int[]) src, toIntIndex(srcOffset / Integer.BYTES), len / Integer.BYTES); - return true; - } else if (src instanceof long[] && aligned(srcOffset, len, Long.BYTES)) { - heapBytes(dst, dstOffset, len) - .asLongBuffer() - .put((long[]) src, toIntIndex(srcOffset / Long.BYTES), len / Long.BYTES); - return true; - } else if (src instanceof float[] && aligned(srcOffset, len, Float.BYTES)) { - heapBytes(dst, dstOffset, len) - .asFloatBuffer() - .put((float[]) src, toIntIndex(srcOffset / Float.BYTES), len / Float.BYTES); - return true; - } else if (src instanceof double[] && aligned(srcOffset, len, Double.BYTES)) { - heapBytes(dst, dstOffset, len) - .asDoubleBuffer() - .put((double[]) src, toIntIndex(srcOffset / Double.BYTES), len / Double.BYTES); - return true; - } - return false; - } - - private boolean copyBytesToArray( - byte[] src, int srcOffset, Object dst, long dstOffset, int len) { - if (dst instanceof boolean[]) { - boolean[] array = (boolean[]) dst; - int dstIndex = toIntIndex(dstOffset); - for (int i = 0; i < len; i++) { - array[dstIndex + i] = src[srcOffset + i] != 0; - } - return true; - } else if (dst instanceof char[] && aligned(dstOffset, len, Character.BYTES)) { - heapBytes(src, srcOffset, len) - .asCharBuffer() - .get((char[]) dst, toIntIndex(dstOffset / Character.BYTES), len / Character.BYTES); - return true; - } else if (dst instanceof short[] && aligned(dstOffset, len, Short.BYTES)) { - heapBytes(src, srcOffset, len) - .asShortBuffer() - .get((short[]) dst, toIntIndex(dstOffset / Short.BYTES), len / Short.BYTES); - return true; - } else if (dst instanceof int[] && aligned(dstOffset, len, Integer.BYTES)) { - heapBytes(src, srcOffset, len) - .asIntBuffer() - .get((int[]) dst, toIntIndex(dstOffset / Integer.BYTES), len / Integer.BYTES); - return true; - } else if (dst instanceof long[] && aligned(dstOffset, len, Long.BYTES)) { - heapBytes(src, srcOffset, len) - .asLongBuffer() - .get((long[]) dst, toIntIndex(dstOffset / Long.BYTES), len / Long.BYTES); - return true; - } else if (dst instanceof float[] && aligned(dstOffset, len, Float.BYTES)) { - heapBytes(src, srcOffset, len) - .asFloatBuffer() - .get((float[]) dst, toIntIndex(dstOffset / Float.BYTES), len / Float.BYTES); - return true; - } else if (dst instanceof double[] && aligned(dstOffset, len, Double.BYTES)) { - heapBytes(src, srcOffset, len) - .asDoubleBuffer() - .get((double[]) dst, toIntIndex(dstOffset / Double.BYTES), len / Double.BYTES); - return true; - } - return false; - } - - private boolean writeArray(Object src, long srcOffset, long dstOffset, int len) { - if (src instanceof boolean[]) { - boolean[] array = (boolean[]) src; - int srcIndex = toIntIndex(srcOffset); - ByteBuffer dst = directBytes(dstOffset, len); - for (int i = 0; i < len; i++) { - dst.put(i, array[srcIndex + i] ? (byte) 1 : (byte) 0); - } - return true; - } else if (src instanceof char[] && aligned(srcOffset, len, Character.BYTES)) { - directBytes(dstOffset, len) - .asCharBuffer() - .put((char[]) src, toIntIndex(srcOffset / Character.BYTES), len / Character.BYTES); - return true; - } else if (src instanceof short[] && aligned(srcOffset, len, Short.BYTES)) { - directBytes(dstOffset, len) - .asShortBuffer() - .put((short[]) src, toIntIndex(srcOffset / Short.BYTES), len / Short.BYTES); - return true; - } else if (src instanceof int[] && aligned(srcOffset, len, Integer.BYTES)) { - directBytes(dstOffset, len) - .asIntBuffer() - .put((int[]) src, toIntIndex(srcOffset / Integer.BYTES), len / Integer.BYTES); - return true; - } else if (src instanceof long[] && aligned(srcOffset, len, Long.BYTES)) { - directBytes(dstOffset, len) - .asLongBuffer() - .put((long[]) src, toIntIndex(srcOffset / Long.BYTES), len / Long.BYTES); - return true; - } else if (src instanceof float[] && aligned(srcOffset, len, Float.BYTES)) { - directBytes(dstOffset, len) - .asFloatBuffer() - .put((float[]) src, toIntIndex(srcOffset / Float.BYTES), len / Float.BYTES); - return true; - } else if (src instanceof double[] && aligned(srcOffset, len, Double.BYTES)) { - directBytes(dstOffset, len) - .asDoubleBuffer() - .put((double[]) src, toIntIndex(srcOffset / Double.BYTES), len / Double.BYTES); - return true; - } - return false; - } - - private boolean readArray(long srcOffset, Object dst, long dstOffset, int len) { - if (dst instanceof boolean[]) { - boolean[] array = (boolean[]) dst; - int dstIndex = toIntIndex(dstOffset); - ByteBuffer src = directBytes(srcOffset, len); - for (int i = 0; i < len; i++) { - array[dstIndex + i] = src.get(i) != 0; - } - return true; - } else if (dst instanceof char[] && aligned(dstOffset, len, Character.BYTES)) { - directBytes(srcOffset, len) - .asCharBuffer() - .get((char[]) dst, toIntIndex(dstOffset / Character.BYTES), len / Character.BYTES); - return true; - } else if (dst instanceof short[] && aligned(dstOffset, len, Short.BYTES)) { - directBytes(srcOffset, len) - .asShortBuffer() - .get((short[]) dst, toIntIndex(dstOffset / Short.BYTES), len / Short.BYTES); - return true; - } else if (dst instanceof int[] && aligned(dstOffset, len, Integer.BYTES)) { - directBytes(srcOffset, len) - .asIntBuffer() - .get((int[]) dst, toIntIndex(dstOffset / Integer.BYTES), len / Integer.BYTES); - return true; - } else if (dst instanceof long[] && aligned(dstOffset, len, Long.BYTES)) { - directBytes(srcOffset, len) - .asLongBuffer() - .get((long[]) dst, toIntIndex(dstOffset / Long.BYTES), len / Long.BYTES); - return true; - } else if (dst instanceof float[] && aligned(dstOffset, len, Float.BYTES)) { - directBytes(srcOffset, len) - .asFloatBuffer() - .get((float[]) dst, toIntIndex(dstOffset / Float.BYTES), len / Float.BYTES); - return true; - } else if (dst instanceof double[] && aligned(dstOffset, len, Double.BYTES)) { - directBytes(srcOffset, len) - .asDoubleBuffer() - .get((double[]) dst, toIntIndex(dstOffset / Double.BYTES), len / Double.BYTES); - return true; - } - return false; - } - - private void copyDirect(long srcOffset, long dstOffset, int len) { - if (srcOffset < dstOffset && dstOffset < srcOffset + len) { - byte[] tmp = new byte[len]; - readDirect(srcOffset, tmp, 0, len); - writeDirect(dstOffset, tmp, 0, len); - } else { - MemorySegment segment = directSegment(); - MemorySegment.copy(segment, srcOffset, segment, dstOffset, len); - } - } - - private ByteBuffer directBuffer() { - ByteBuffer directBuffer = buffer.nativeOffHeapBuffer; - if (directBuffer == null) { - throw new IllegalStateException("Memory buffer does not own a ByteBuffer"); - } - return directBuffer; - } - - private MemorySegment directSegment() { - MemorySegment segment = buffer.offHeapSegment; - if (segment == null) { - throw new IllegalStateException("Memory buffer does not own an off-heap segment"); - } - return segment; - } - - private void readDirect(long offset, byte[] dst, int dstOffset, int length) { - directBuffer().get(toIntIndex(offset), dst, dstOffset, length); - } - - private void writeDirect(long offset, byte[] src, int srcOffset, int length) { - directBuffer().put(toIntIndex(offset), src, srcOffset, length); - } - - private ByteBuffer directBytes(long offset, int length) { - ByteBuffer duplicate = directBuffer().duplicate().order(NATIVE_ORDER); - int start = toIntIndex(offset); - ByteBufferUtil.position(duplicate, start); - duplicate.limit(start + length); - return duplicate.slice().order(NATIVE_ORDER); - } - - private static ByteBuffer heapBytes(byte[] bytes, int offset, int length) { - return ByteBuffer.wrap(bytes, offset, length).order(NATIVE_ORDER); - } - - private static boolean aligned(long offset, int length, int width) { - return offset % width == 0 && length % width == 0; - } - - private static int toIntIndex(long offset) { - if (offset < 0 || offset > Integer.MAX_VALUE) { - throw new IndexOutOfBoundsException("offset out of int range: " + offset); - } - return (int) offset; + private byte rawByte(Object base, long offset) { + if (base == null) { + return loadByte(offset); } + return ((byte[]) base)[toIntIndex(offset)]; + } - private static int toIntLength(long length) { - if (length < 0 || length > Integer.MAX_VALUE) { - throw new IndexOutOfBoundsException("length out of int range: " + length); - } - return (int) length; + private long rawLong(Object base, long offset) { + if (base == null) { + return loadLong(offset); } + return (long) BYTE_ARRAY_LONG.get((byte[]) base, toIntIndex(offset)); } @Override diff --git a/java/fory-core/src/main/java25/org/apache/fory/platform/UnsafeOps.java b/java/fory-core/src/main/java25/org/apache/fory/platform/UnsafeOps.java deleted file mode 100644 index cd478704d3..0000000000 --- a/java/fory-core/src/main/java25/org/apache/fory/platform/UnsafeOps.java +++ /dev/null @@ -1,497 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.fory.platform; - -import java.lang.invoke.MethodHandles; -import java.lang.invoke.VarHandle; -import java.lang.reflect.Field; -import java.nio.ByteOrder; -import org.apache.fory.annotation.Internal; -import org.apache.fory.platform.internal._JDKAccess; -import sun.misc.Unsafe; - -/** A utility class for array memory operations on JDK25+. */ -@Internal -@SuppressWarnings("restriction") -public final class UnsafeOps { - @SuppressWarnings("restriction") - public static final Unsafe UNSAFE = _JDKAccess.UNSAFE; - - // JDK25 array operations use Java/VarHandle indexes instead of raw Unsafe byte offsets. - // Keep these constants zero so versioned MemoryBuffer code can preserve the root API shape - // without mixing Unsafe base-offset domains into the zero-Unsafe runtime. - public static final int BOOLEAN_ARRAY_OFFSET = 0; - public static final int BYTE_ARRAY_OFFSET = 0; - public static final int CHAR_ARRAY_OFFSET = 0; - public static final int SHORT_ARRAY_OFFSET = 0; - public static final int INT_ARRAY_OFFSET = 0; - public static final int LONG_ARRAY_OFFSET = 0; - public static final int FLOAT_ARRAY_OFFSET = 0; - public static final int DOUBLE_ARRAY_OFFSET = 0; - private static final boolean BIG_ENDIAN = ByteOrder.nativeOrder() == ByteOrder.BIG_ENDIAN; - private static final VarHandle BYTE_ARRAY_CHAR = - MethodHandles.byteArrayViewVarHandle(char[].class, ByteOrder.nativeOrder()); - private static final VarHandle BYTE_ARRAY_SHORT = - MethodHandles.byteArrayViewVarHandle(short[].class, ByteOrder.nativeOrder()); - private static final VarHandle BYTE_ARRAY_INT = - MethodHandles.byteArrayViewVarHandle(int[].class, ByteOrder.nativeOrder()); - private static final VarHandle BYTE_ARRAY_LONG = - MethodHandles.byteArrayViewVarHandle(long[].class, ByteOrder.nativeOrder()); - private static final VarHandle BYTE_ARRAY_FLOAT = - MethodHandles.byteArrayViewVarHandle(float[].class, ByteOrder.nativeOrder()); - private static final VarHandle BYTE_ARRAY_DOUBLE = - MethodHandles.byteArrayViewVarHandle(double[].class, ByteOrder.nativeOrder()); - private static final boolean unaligned; - - private UnsafeOps() {} - - static { - String arch = System.getProperty("os.arch", ""); - if ("ppc64le".equals(arch) || "ppc64".equals(arch) || "s390x".equals(arch)) { - unaligned = true; - } else { - unaligned = arch.matches("^(i[3-6]86|x86(_64)?|x64|amd64|aarch64)$"); - } - } - - /** - * Returns true when the underlying system is known to support unaligned access. JDK25 array - * accessors do not use Unsafe, but callers keep this gate for vectorized array scans. - */ - public static boolean unaligned() { - return unaligned; - } - - public static long objectFieldOffset(Field f) { - throw unsupportedObjectMemory(); - } - - public static int getInt(Object object, long offset) { - if (object instanceof byte[]) { - return (int) BYTE_ARRAY_INT.get((byte[]) object, toIntIndex(offset)); - } - return getIntFromArray(object, offset); - } - - public static void putInt(Object object, long offset, int value) { - if (object instanceof byte[]) { - BYTE_ARRAY_INT.set((byte[]) object, toIntIndex(offset), value); - return; - } - putIntToArray(object, offset, value); - } - - public static boolean getBoolean(Object object, long offset) { - if (object instanceof boolean[]) { - return ((boolean[]) object)[toIntIndex(offset)]; - } - return getByte(object, offset) != 0; - } - - public static void putBoolean(Object object, long offset, boolean value) { - if (object instanceof boolean[]) { - ((boolean[]) object)[toIntIndex(offset)] = value; - return; - } - putByte(object, offset, value ? (byte) 1 : (byte) 0); - } - - public static byte getByte(Object object, long offset) { - return getArrayByte(object, offset); - } - - public static void putByte(Object object, long offset, byte value) { - putArrayByte(object, offset, value); - } - - public static short getShort(Object object, long offset) { - if (object instanceof byte[]) { - return (short) BYTE_ARRAY_SHORT.get((byte[]) object, toIntIndex(offset)); - } - return (short) getIntN(object, offset, Short.BYTES); - } - - public static void putShort(Object object, long offset, short value) { - if (object instanceof byte[]) { - BYTE_ARRAY_SHORT.set((byte[]) object, toIntIndex(offset), value); - return; - } - putIntN(object, offset, value, Short.BYTES); - } - - public static char getChar(Object obj, long offset) { - if (obj instanceof byte[]) { - return (char) BYTE_ARRAY_CHAR.get((byte[]) obj, toIntIndex(offset)); - } - return (char) getIntN(obj, offset, Character.BYTES); - } - - public static void putChar(Object obj, long offset, char value) { - if (obj instanceof byte[]) { - BYTE_ARRAY_CHAR.set((byte[]) obj, toIntIndex(offset), value); - return; - } - putIntN(obj, offset, value, Character.BYTES); - } - - public static long getLong(Object object, long offset) { - if (object instanceof byte[]) { - return (long) BYTE_ARRAY_LONG.get((byte[]) object, toIntIndex(offset)); - } - long value = 0; - if (BIG_ENDIAN) { - for (int i = 0; i < Long.BYTES; i++) { - value = (value << Byte.SIZE) | (getArrayByte(object, offset + i) & 0xffL); - } - } else { - for (int i = Long.BYTES - 1; i >= 0; i--) { - value = (value << Byte.SIZE) | (getArrayByte(object, offset + i) & 0xffL); - } - } - return value; - } - - public static void putLong(Object object, long offset, long value) { - if (object instanceof byte[]) { - BYTE_ARRAY_LONG.set((byte[]) object, toIntIndex(offset), value); - return; - } - if (BIG_ENDIAN) { - for (int i = Long.BYTES - 1; i >= 0; i--) { - putArrayByte(object, offset + i, (byte) value); - value >>>= Byte.SIZE; - } - } else { - for (int i = 0; i < Long.BYTES; i++) { - putArrayByte(object, offset + i, (byte) value); - value >>>= Byte.SIZE; - } - } - } - - public static float getFloat(Object object, long offset) { - if (object instanceof byte[]) { - return (float) BYTE_ARRAY_FLOAT.get((byte[]) object, toIntIndex(offset)); - } - return Float.intBitsToFloat(getInt(object, offset)); - } - - public static void putFloat(Object object, long offset, float value) { - if (object instanceof byte[]) { - BYTE_ARRAY_FLOAT.set((byte[]) object, toIntIndex(offset), value); - return; - } - putInt(object, offset, Float.floatToRawIntBits(value)); - } - - public static double getDouble(Object object, long offset) { - if (object instanceof byte[]) { - return (double) BYTE_ARRAY_DOUBLE.get((byte[]) object, toIntIndex(offset)); - } - return Double.longBitsToDouble(getLong(object, offset)); - } - - public static void putDouble(Object object, long offset, double value) { - if (object instanceof byte[]) { - BYTE_ARRAY_DOUBLE.set((byte[]) object, toIntIndex(offset), value); - return; - } - putLong(object, offset, Double.doubleToRawLongBits(value)); - } - - public static Object getObject(Object o, long offset) { - throw unsupportedObjectMemory(); - } - - public static void putObject(Object object, long offset, Object value) { - throw unsupportedObjectMemory(); - } - - public static void copyMemory( - Object src, long srcOffset, Object dst, long dstOffset, long length) { - if (src == null || dst == null) { - throw unsupportedNativeMemory(); - } - if (length < 0) { - throw new IllegalArgumentException("length must be non-negative: " + length); - } - int len = toIntLength(length); - if (src instanceof byte[] && dst instanceof byte[]) { - System.arraycopy((byte[]) src, toIntIndex(srcOffset), (byte[]) dst, toIntIndex(dstOffset), len); - return; - } - if (copySamePrimitiveArray(src, srcOffset, dst, dstOffset, len)) { - return; - } - if (!isPrimitiveArray(src) || !isPrimitiveArray(dst)) { - throw unsupportedObjectMemory(); - } - if (src == dst && srcOffset < dstOffset && dstOffset < srcOffset + length) { - for (long i = length - 1; i >= 0; i--) { - putArrayByte(dst, dstOffset + i, getArrayByte(src, srcOffset + i)); - } - } else { - for (long i = 0; i < length; i++) { - putArrayByte(dst, dstOffset + i, getArrayByte(src, srcOffset + i)); - } - } - } - - private static boolean copySamePrimitiveArray( - Object src, long srcOffset, Object dst, long dstOffset, int len) { - if (src.getClass() != dst.getClass()) { - return false; - } - if (src instanceof boolean[]) { - System.arraycopy((boolean[]) src, toIntIndex(srcOffset), (boolean[]) dst, toIntIndex(dstOffset), len); - return true; - } else if (src instanceof char[] && aligned(srcOffset, dstOffset, len, Character.BYTES)) { - System.arraycopy( - (char[]) src, - toIntIndex(srcOffset / Character.BYTES), - (char[]) dst, - toIntIndex(dstOffset / Character.BYTES), - len / Character.BYTES); - return true; - } else if (src instanceof short[] && aligned(srcOffset, dstOffset, len, Short.BYTES)) { - System.arraycopy( - (short[]) src, - toIntIndex(srcOffset / Short.BYTES), - (short[]) dst, - toIntIndex(dstOffset / Short.BYTES), - len / Short.BYTES); - return true; - } else if (src instanceof int[] && aligned(srcOffset, dstOffset, len, Integer.BYTES)) { - System.arraycopy( - (int[]) src, - toIntIndex(srcOffset / Integer.BYTES), - (int[]) dst, - toIntIndex(dstOffset / Integer.BYTES), - len / Integer.BYTES); - return true; - } else if (src instanceof long[] && aligned(srcOffset, dstOffset, len, Long.BYTES)) { - System.arraycopy( - (long[]) src, - toIntIndex(srcOffset / Long.BYTES), - (long[]) dst, - toIntIndex(dstOffset / Long.BYTES), - len / Long.BYTES); - return true; - } else if (src instanceof float[] && aligned(srcOffset, dstOffset, len, Float.BYTES)) { - System.arraycopy( - (float[]) src, - toIntIndex(srcOffset / Float.BYTES), - (float[]) dst, - toIntIndex(dstOffset / Float.BYTES), - len / Float.BYTES); - return true; - } else if (src instanceof double[] && aligned(srcOffset, dstOffset, len, Double.BYTES)) { - System.arraycopy( - (double[]) src, - toIntIndex(srcOffset / Double.BYTES), - (double[]) dst, - toIntIndex(dstOffset / Double.BYTES), - len / Double.BYTES); - return true; - } - return false; - } - - public static Object[] copyObjectArray(Object[] arr) { - Object[] objects = new Object[arr.length]; - System.arraycopy(arr, 0, objects, 0, arr.length); - return objects; - } - - /** Create an instance of type. This method does not call constructor. */ - public static T newInstance(Class type) { - throw new UnsupportedOperationException( - "Constructor-bypassing allocation is unsupported on JDK25 without sun.misc.Unsafe; " - + "use a constructor-based serializer path for " - + type); - } - - private static int getIntFromArray(Object object, long offset) { - return getIntN(object, offset, Integer.BYTES); - } - - private static void putIntToArray(Object object, long offset, int value) { - putIntN(object, offset, value, Integer.BYTES); - } - - private static int getIntN(Object object, long offset, int bytes) { - int value = 0; - if (BIG_ENDIAN) { - for (int i = 0; i < bytes; i++) { - value = (value << Byte.SIZE) | (getArrayByte(object, offset + i) & 0xff); - } - } else { - for (int i = bytes - 1; i >= 0; i--) { - value = (value << Byte.SIZE) | (getArrayByte(object, offset + i) & 0xff); - } - } - return value; - } - - private static void putIntN(Object object, long offset, int value, int bytes) { - if (BIG_ENDIAN) { - for (int i = bytes - 1; i >= 0; i--) { - putArrayByte(object, offset + i, (byte) value); - value >>>= Byte.SIZE; - } - } else { - for (int i = 0; i < bytes; i++) { - putArrayByte(object, offset + i, (byte) value); - value >>>= Byte.SIZE; - } - } - } - - private static byte getArrayByte(Object object, long offset) { - checkOffset(offset); - if (object instanceof byte[]) { - return ((byte[]) object)[toIntIndex(offset)]; - } else if (object instanceof boolean[]) { - return ((boolean[]) object)[toIntIndex(offset)] ? (byte) 1 : (byte) 0; - } else if (object instanceof char[]) { - return getIntByte( - ((char[]) object)[toIntIndex(offset / Character.BYTES)], offset, Character.BYTES); - } else if (object instanceof short[]) { - return getIntByte(((short[]) object)[toIntIndex(offset / Short.BYTES)], offset, Short.BYTES); - } else if (object instanceof int[]) { - return getIntByte( - ((int[]) object)[toIntIndex(offset / Integer.BYTES)], offset, Integer.BYTES); - } else if (object instanceof long[]) { - return getLongByte(((long[]) object)[toIntIndex(offset / Long.BYTES)], offset); - } else if (object instanceof float[]) { - int value = Float.floatToRawIntBits(((float[]) object)[toIntIndex(offset / Float.BYTES)]); - return getIntByte(value, offset, Float.BYTES); - } else if (object instanceof double[]) { - long value = - Double.doubleToRawLongBits(((double[]) object)[toIntIndex(offset / Double.BYTES)]); - return getLongByte(value, offset); - } - throw unsupportedObjectMemory(); - } - - private static void putArrayByte(Object object, long offset, byte value) { - checkOffset(offset); - if (object instanceof byte[]) { - ((byte[]) object)[toIntIndex(offset)] = value; - } else if (object instanceof boolean[]) { - ((boolean[]) object)[toIntIndex(offset)] = value != 0; - } else if (object instanceof char[]) { - char[] array = (char[]) object; - int index = toIntIndex(offset / Character.BYTES); - array[index] = (char) setIntByte(array[index], offset, value, Character.BYTES); - } else if (object instanceof short[]) { - short[] array = (short[]) object; - int index = toIntIndex(offset / Short.BYTES); - array[index] = (short) setIntByte(array[index], offset, value, Short.BYTES); - } else if (object instanceof int[]) { - int[] array = (int[]) object; - int index = toIntIndex(offset / Integer.BYTES); - array[index] = setIntByte(array[index], offset, value, Integer.BYTES); - } else if (object instanceof long[]) { - long[] array = (long[]) object; - int index = toIntIndex(offset / Long.BYTES); - array[index] = setLongByte(array[index], offset, value); - } else if (object instanceof float[]) { - float[] array = (float[]) object; - int index = toIntIndex(offset / Float.BYTES); - int bits = Float.floatToRawIntBits(array[index]); - array[index] = Float.intBitsToFloat(setIntByte(bits, offset, value, Float.BYTES)); - } else if (object instanceof double[]) { - double[] array = (double[]) object; - int index = toIntIndex(offset / Double.BYTES); - long bits = Double.doubleToRawLongBits(array[index]); - array[index] = Double.longBitsToDouble(setLongByte(bits, offset, value)); - } else { - throw unsupportedObjectMemory(); - } - } - - private static byte getIntByte(int value, long offset, int width) { - int shift = byteShift(offset, width); - return (byte) (value >>> shift); - } - - private static byte getLongByte(long value, long offset) { - int shift = byteShift(offset, Long.BYTES); - return (byte) (value >>> shift); - } - - private static int setIntByte(int oldValue, long offset, byte value, int width) { - int shift = byteShift(offset, width); - int mask = 0xff << shift; - return (oldValue & ~mask) | ((value & 0xff) << shift); - } - - private static long setLongByte(long oldValue, long offset, byte value) { - int shift = byteShift(offset, Long.BYTES); - long mask = 0xffL << shift; - return (oldValue & ~mask) | ((long) (value & 0xff) << shift); - } - - private static int byteShift(long offset, int width) { - int byteIndex = (int) Math.floorMod(offset, width); - return (BIG_ENDIAN ? width - 1 - byteIndex : byteIndex) * Byte.SIZE; - } - - private static boolean isPrimitiveArray(Object object) { - Class cls = object.getClass(); - return cls.isArray() && cls.getComponentType().isPrimitive(); - } - - private static boolean aligned(long srcOffset, long dstOffset, int len, int width) { - return srcOffset % width == 0 && dstOffset % width == 0 && len % width == 0; - } - - private static int toIntIndex(long offset) { - if (offset < 0 || offset > Integer.MAX_VALUE) { - throw new IndexOutOfBoundsException("offset out of int range: " + offset); - } - return (int) offset; - } - - private static int toIntLength(long length) { - if (length > Integer.MAX_VALUE) { - throw new IndexOutOfBoundsException("length out of int range: " + length); - } - return (int) length; - } - - private static void checkOffset(long offset) { - if (offset < 0) { - throw new IndexOutOfBoundsException("offset must be non-negative: " + offset); - } - } - - private static UnsupportedOperationException unsupportedObjectMemory() { - return new UnsupportedOperationException( - "Object field and reference-offset memory access is unsupported on JDK25 without " - + "sun.misc.Unsafe"); - } - - private static UnsupportedOperationException unsupportedNativeMemory() { - return new UnsupportedOperationException( - "Raw native-address memory access is unsupported on JDK25 without sun.misc.Unsafe"); - } -} diff --git a/java/fory-core/src/main/java25/org/apache/fory/platform/internal/_JDKAccess.java b/java/fory-core/src/main/java25/org/apache/fory/platform/internal/_JDKAccess.java index 0c057eb336..56caf8e197 100644 --- a/java/fory-core/src/main/java25/org/apache/fory/platform/internal/_JDKAccess.java +++ b/java/fory-core/src/main/java25/org/apache/fory/platform/internal/_JDKAccess.java @@ -78,6 +78,10 @@ public class _JDKAccess { public static final Class _INNER_UNSAFE_CLASS = null; public static final Object _INNER_UNSAFE = null; + public static Unsafe unsafe() { + return UNSAFE; + } + private static final ClassValueCache lookupCache = ClassValueCache.newClassKeyCache(32); public static final boolean STRING_VALUE_FIELD_IS_CHARS; diff --git a/java/fory-core/src/main/resources/META-INF/LICENSE b/java/fory-core/src/main/resources/META-INF/LICENSE index 6de997e32b..149d74bdcf 100644 --- a/java/fory-core/src/main/resources/META-INF/LICENSE +++ b/java/fory-core/src/main/resources/META-INF/LICENSE @@ -224,7 +224,6 @@ The text of each license is the standard Apache 2.0 license. * spark (https://github.com/apache/spark) Files: java/fory-core/src/main/java/org/apache/fory/codegen/Code.java - java/fory-core/src/main/java/org/apache/fory/platform/UnsafeOps.java * commons-io (https://github.com/apache/commons-io) Files: diff --git a/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties b/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties index 2995057ccd..56c9c45adc 100644 --- a/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties +++ b/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties @@ -35,6 +35,7 @@ Args=--initialize-at-build-time=org.apache.fory.annotation.ForyField$Dynamic,\ org.apache.fory.serializer.collection.GuavaCollectionSerializers,\ org.apache.fory.serializer.struct.Fingerprint,\ org.apache.fory.serializer.Float16Serializer,\ + org.apache.fory.serializer.PlatformStringUtils,\ org.apache.fory.serializer.PrimitiveSerializers$Float16Serializer,\ java.io.IOException,\ java.lang.ArrayIndexOutOfBoundsException,\ @@ -212,7 +213,6 @@ Args=--initialize-at-build-time=org.apache.fory.annotation.ForyField$Dynamic,\ org.apache.fory.memory.MemoryBuffer$DefaultMemoryAllocator,\ org.apache.fory.memory.MemoryUtils,\ org.apache.fory.platform.JdkVersion,\ - org.apache.fory.platform.UnsafeOps,\ org.apache.fory.collection.ConcurrentIdentityMap,\ org.apache.fory.collection.ConcurrentIdentityMap$IdentityKey,\ org.apache.fory.meta.FieldTypes,\ diff --git a/java/fory-core/src/test/java/org/apache/fory/GuavaOptionalDependencyTest.java b/java/fory-core/src/test/java/org/apache/fory/GuavaOptionalDependencyTest.java index a10e9270db..476caef205 100644 --- a/java/fory-core/src/test/java/org/apache/fory/GuavaOptionalDependencyTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/GuavaOptionalDependencyTest.java @@ -66,11 +66,9 @@ private static int registeredInternalId(boolean registerGuavaTypes) { } private static RegistrationIds runWithoutGuava() throws Exception { - String javaBin = - System.getProperty("java.home") + File.separator + "bin" + File.separator + "java"; String filteredClassPath = removeGuavaFromClasspath(System.getProperty("java.class.path")); Process process = - new ProcessBuilder(javaBin, "-cp", filteredClassPath, NoGuavaMain.class.getName()) + new ProcessBuilder(TestUtils.javaCommand(filteredClassPath, NoGuavaMain.class)) .redirectErrorStream(true) .start(); String output = readFully(process.getInputStream()); diff --git a/java/fory-core/src/test/java/org/apache/fory/JpmsOptionalClassLoadingTest.java b/java/fory-core/src/test/java/org/apache/fory/JpmsOptionalClassLoadingTest.java index 3a14548cd0..ee7971e7fe 100644 --- a/java/fory-core/src/test/java/org/apache/fory/JpmsOptionalClassLoadingTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/JpmsOptionalClassLoadingTest.java @@ -22,7 +22,6 @@ import static org.testng.Assert.assertEquals; import java.io.ByteArrayOutputStream; -import java.io.File; import java.io.IOException; import java.io.InputStream; import java.io.Serializable; @@ -37,16 +36,13 @@ public void testBuildWithoutJavaSqlModule() throws Exception { if (JdkVersion.MAJOR_VERSION < 9) { throw new SkipException("Skip on jdk" + JdkVersion.MAJOR_VERSION); } - String javaBin = - System.getProperty("java.home") + File.separator + "bin" + File.separator + "java"; Process process = new ProcessBuilder( - javaBin, - "--limit-modules", - "java.base,java.logging,jdk.unsupported", - "-cp", - System.getProperty("java.class.path"), - NoJavaSqlMain.class.getName()) + TestUtils.javaCommand( + System.getProperty("java.class.path"), + NoJavaSqlMain.class, + "--limit-modules", + "java.base,java.logging,jdk.unsupported")) .redirectErrorStream(true) .start(); String output = readFully(process.getInputStream()); diff --git a/java/fory-core/src/test/java/org/apache/fory/StreamTest.java b/java/fory-core/src/test/java/org/apache/fory/StreamTest.java index e25b486493..41e63fa474 100644 --- a/java/fory-core/src/test/java/org/apache/fory/StreamTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/StreamTest.java @@ -457,6 +457,15 @@ public void testPrimitiveArrayStreamReaderUsesTypedReads() throws IOException { Assert.assertEquals((long[]) fory.deserialize(channel), longs); assertTrue(channel.readLongsCalled); } + + ByteBuffer limitedDirectBuffer = ByteBuffer.allocateDirect(serialized.length + 8); + limitedDirectBuffer.limit(5); + try (TrackingForyReadableChannel channel = + new TrackingForyReadableChannel( + new ChunkedReadableByteChannel(serialized, 1), limitedDirectBuffer)) { + Assert.assertEquals((long[]) fory.deserialize(channel), longs); + assertTrue(channel.readLongsCalled); + } } @Test diff --git a/java/fory-core/src/test/java/org/apache/fory/TestUtils.java b/java/fory-core/src/test/java/org/apache/fory/TestUtils.java index 2a2e5f1e19..67ed2b8823 100644 --- a/java/fory-core/src/test/java/org/apache/fory/TestUtils.java +++ b/java/fory-core/src/test/java/org/apache/fory/TestUtils.java @@ -22,8 +22,10 @@ import com.google.common.collect.ImmutableBiMap; import com.google.common.collect.ImmutableMap; import java.io.ByteArrayOutputStream; +import java.io.File; import java.io.IOException; import java.io.ObjectOutputStream; +import java.lang.management.ManagementFactory; import java.lang.reflect.Field; import java.lang.reflect.Modifier; import java.util.*; @@ -31,15 +33,56 @@ import java.util.stream.Collectors; import org.apache.fory.collection.Tuple3; import org.apache.fory.meta.TypeDef; -import org.apache.fory.platform.UnsafeOps; +import org.apache.fory.platform.JdkVersion; import org.apache.fory.platform.internal._JDKAccess; import org.apache.fory.reflect.FieldAccessor; +import org.apache.fory.reflect.ObjectCreators; import org.apache.fory.reflect.ReflectionUtils; import org.apache.fory.type.Descriptor; import org.testng.SkipException; /** Test utils. */ public class TestUtils { + public static List javaCommand(Class mainClass) { + return javaCommand(System.getProperty("java.class.path"), mainClass); + } + + public static List javaCommand( + String classPath, Class mainClass, String... extraJvmArgs) { + List command = new ArrayList<>(); + command.add(System.getProperty("java.home") + File.separator + "bin" + File.separator + "java"); + command.addAll(forkJvmArgs()); + Collections.addAll(command, extraJvmArgs); + command.add("-cp"); + command.add(classPath); + command.add(mainClass.getName()); + return command; + } + + private static List forkJvmArgs() { + List args = new ArrayList<>(); + if (JdkVersion.MAJOR_VERSION >= 25) { + args.add("--add-opens=java.base/java.lang=ALL-UNNAMED"); + args.add("--add-opens=java.base/java.lang.invoke=ALL-UNNAMED"); + args.add("--add-opens=java.base/java.lang.reflect=ALL-UNNAMED"); + args.add("--add-opens=java.base/jdk.internal.reflect=ALL-UNNAMED"); + args.add("--add-opens=java.base/java.util=ALL-UNNAMED"); + args.add("--add-opens=java.base/java.util.concurrent=ALL-UNNAMED"); + args.add("--add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED"); + args.add("--add-opens=java.base/java.io=ALL-UNNAMED"); + args.add("--add-opens=java.base/java.net=ALL-UNNAMED"); + args.add("--add-opens=java.base/java.math=ALL-UNNAMED"); + if (hasInputArg("--sun-misc-unsafe-memory-access=deny")) { + args.add("--sun-misc-unsafe-memory-access=deny"); + } + } + return args; + } + + private static boolean hasInputArg(String arg) { + return ManagementFactory.getRuntimeMXBean().getInputArguments().contains(arg); + } + @SuppressWarnings("unchecked") public static T getFieldValue(Object obj, String fieldName) { return (T) @@ -106,7 +149,7 @@ public static void jdkSerialize(ByteArrayOutputStream bas, Object data) { public static T unsafeCopy(T obj) { @SuppressWarnings("unchecked") - T newInstance = (T) UnsafeOps.newInstance(obj.getClass()); + T newInstance = (T) ObjectCreators.getObjectCreator(obj.getClass()).newInstance(); for (Field field : ReflectionUtils.getFields(obj.getClass(), true)) { if (!Modifier.isStatic(field.getModifiers())) { // Don't cache accessors by `obj.getClass()` using WeakHashMap, the `field` will reference diff --git a/java/fory-core/src/test/java/org/apache/fory/config/ForyBuilderTest.java b/java/fory-core/src/test/java/org/apache/fory/config/ForyBuilderTest.java index c72cfd02a6..6b552966c5 100644 --- a/java/fory-core/src/test/java/org/apache/fory/config/ForyBuilderTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/config/ForyBuilderTest.java @@ -24,11 +24,11 @@ import static org.testng.Assert.assertTrue; import java.io.ByteArrayOutputStream; -import java.io.File; import java.io.IOException; import java.io.InputStream; import java.nio.charset.StandardCharsets; import org.apache.fory.Fory; +import org.apache.fory.TestUtils; import org.apache.fory.ThreadSafeFory; import org.apache.fory.meta.MetaCompressor; import org.testng.Assert; @@ -155,15 +155,12 @@ public void testCodegenDefaultsOnOrdinaryJvm() { @Test public void testGraalvmRuntimeForcesCodegenOff() throws Exception { - String javaBin = - System.getProperty("java.home") + File.separator + "bin" + File.separator + "java"; Process process = new ProcessBuilder( - javaBin, - "-Dorg.graalvm.nativeimage.imagecode=runtime", - "-cp", - System.getProperty("java.class.path"), - GraalvmCodegenConfigMain.class.getName()) + TestUtils.javaCommand( + System.getProperty("java.class.path"), + GraalvmCodegenConfigMain.class, + "-Dorg.graalvm.nativeimage.imagecode=runtime")) .redirectErrorStream(true) .start(); String output = readFully(process.getInputStream()); diff --git a/java/fory-core/src/test/java/org/apache/fory/memory/MemoryBufferTest.java b/java/fory-core/src/test/java/org/apache/fory/memory/MemoryBufferTest.java index f395e60467..4a30f92a55 100644 --- a/java/fory-core/src/test/java/org/apache/fory/memory/MemoryBufferTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/memory/MemoryBufferTest.java @@ -31,6 +31,7 @@ import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.util.Random; +import org.apache.fory.TestUtils; import org.apache.fory.platform.AndroidSupport; import org.apache.fory.platform.JdkVersion; import org.testng.Assert; @@ -114,6 +115,22 @@ public void testFromDirectByteBufferRejectsHeapBuffer() { () -> MemoryBuffer.fromDirectByteBuffer(ByteBuffer.allocate(8), 8, null)); } + @Test + public void testDirectByteBufferNoNioOpen() throws Exception { + ProcessBuilder processBuilder = + new ProcessBuilder(TestUtils.javaCommand(DirectByteBufferNoNioOpenProbe.class)) + .redirectErrorStream(true); + for (String commandPart : processBuilder.command()) { + assertTrue(!commandPart.contains("java.base/java.nio"), processBuilder.command().toString()); + } + processBuilder.environment().remove("JDK_JAVA_OPTIONS"); + processBuilder.environment().remove("JAVA_TOOL_OPTIONS"); + processBuilder.environment().remove("_JAVA_OPTIONS"); + Process process = processBuilder.start(); + String output = readFully(process.getInputStream()); + assertEquals(process.waitFor(), 0, output); + } + @Test public void testAndroidHeapMemoryBufferPaths() throws Exception { String javaBin = @@ -218,8 +235,8 @@ public static void main(String[] args) { byte[] bytes = new byte[4]; source.copyToByteArray(0, bytes, 0, 4); check(bytes, new byte[] {1, 2, 3, 4}); - assertThrows( - UnsupportedOperationException.class, () -> target.copyFromUnsafe(0, new byte[4], 0, 4)); + target.copyFromByteArray(0, new byte[] {4, 3, 2, 1}, 0, 4); + check(target.getBytes(0, 4), new byte[] {4, 3, 2, 1}); } private static void check(boolean actual, boolean expected) { @@ -281,6 +298,54 @@ private static void check(byte[] actual, byte[] expected) { } } + public static final class DirectByteBufferNoNioOpenProbe { + public static void main(String[] args) { + if (JdkVersion.MAJOR_VERSION >= 25) { + for (String inputArg : + java.lang.management.ManagementFactory.getRuntimeMXBean().getInputArguments()) { + if (inputArg.contains("java.base/java.nio")) { + throw new AssertionError("Unexpected java.nio open: " + inputArg); + } + } + if (ByteBuffer.class + .getModule() + .isOpen("java.nio", DirectByteBufferNoNioOpenProbe.class.getModule())) { + throw new AssertionError("java.base/java.nio must not be open to this test probe"); + } + } + MemoryBuffer buffer = MemoryUtils.wrap(ByteBuffer.allocateDirect(128)); + buffer.writeInt32(17); + buffer.writeInt64(19); + checkEqual(buffer.readInt32(), 17); + checkEqual(buffer.readInt64(), 19L); + + int[] ints = new int[] {1, 2, 3, 4}; + buffer.writerIndex(0); + buffer.copyFromIntArray(0, ints, 0, ints.length * Integer.BYTES); + int[] readInts = new int[ints.length]; + buffer.copyToIntArray(0, readInts, 0, readInts.length * Integer.BYTES); + if (!java.util.Arrays.equals(readInts, ints)) { + throw new AssertionError( + "Expected " + + java.util.Arrays.toString(ints) + + " but got " + + java.util.Arrays.toString(readInts)); + } + } + + private static void checkEqual(int actual, int expected) { + if (actual != expected) { + throw new AssertionError("Expected " + expected + " but got " + actual); + } + } + + private static void checkEqual(long actual, long expected) { + if (actual != expected) { + throw new AssertionError("Expected " + expected + " but got " + actual); + } + } + } + @Test public void testBufferUnsafeWrite() { { diff --git a/java/fory-core/src/test/java/org/apache/fory/resolver/GraalvmRuntimeArrayTest.java b/java/fory-core/src/test/java/org/apache/fory/resolver/GraalvmRuntimeArrayTest.java index 023796ea1e..94270d60bf 100644 --- a/java/fory-core/src/test/java/org/apache/fory/resolver/GraalvmRuntimeArrayTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/resolver/GraalvmRuntimeArrayTest.java @@ -22,26 +22,23 @@ import static org.testng.Assert.assertEquals; import java.io.ByteArrayOutputStream; -import java.io.File; import java.io.IOException; import java.io.InputStream; import java.nio.charset.StandardCharsets; import org.apache.fory.Fory; +import org.apache.fory.TestUtils; import org.apache.fory.serializer.ArraySerializers; import org.testng.annotations.Test; public class GraalvmRuntimeArrayTest { @Test public void testGraalvmRuntimeFallsBackForUnregisteredArrayClass() throws Exception { - String javaBin = - System.getProperty("java.home") + File.separator + "bin" + File.separator + "java"; Process process = new ProcessBuilder( - javaBin, - "-Dorg.graalvm.nativeimage.imagecode=runtime", - "-cp", - System.getProperty("java.class.path"), - GraalvmRuntimeArrayMain.class.getName()) + TestUtils.javaCommand( + System.getProperty("java.class.path"), + GraalvmRuntimeArrayMain.class, + "-Dorg.graalvm.nativeimage.imagecode=runtime")) .redirectErrorStream(true) .start(); String output = readFully(process.getInputStream()); diff --git a/java/fory-core/src/test/java/org/apache/fory/serializer/AndroidDynamicFeatureTest.java b/java/fory-core/src/test/java/org/apache/fory/serializer/AndroidDynamicFeatureTest.java index 667de876ae..990f09a730 100644 --- a/java/fory-core/src/test/java/org/apache/fory/serializer/AndroidDynamicFeatureTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/serializer/AndroidDynamicFeatureTest.java @@ -21,7 +21,6 @@ import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; -import java.io.File; import java.io.IOException; import java.io.InputStream; import java.io.ObjectInputStream; @@ -37,6 +36,7 @@ import java.util.Set; import java.util.function.ToIntFunction; import org.apache.fory.Fory; +import org.apache.fory.TestUtils; import org.apache.fory.context.ReadContext; import org.apache.fory.context.WriteContext; import org.apache.fory.memory.MemoryUtils; @@ -54,14 +54,8 @@ public class AndroidDynamicFeatureTest { @Test public void testAndroidDynamicFeaturePaths() throws Exception { - String javaBin = - System.getProperty("java.home") + File.separator + "bin" + File.separator + "java"; Process process = - new ProcessBuilder( - javaBin, - "-cp", - System.getProperty("java.class.path"), - AndroidDynamicFeatureProbe.class.getName()) + new ProcessBuilder(TestUtils.javaCommand(AndroidDynamicFeatureProbe.class)) .redirectErrorStream(true) .start(); String output = readFully(process.getInputStream()); diff --git a/java/fory-core/src/test/java/org/apache/fory/util/StringUtilsTest.java b/java/fory-core/src/test/java/org/apache/fory/util/StringUtilsTest.java index 4d639a4236..2fd109cb67 100644 --- a/java/fory-core/src/test/java/org/apache/fory/util/StringUtilsTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/util/StringUtilsTest.java @@ -24,13 +24,10 @@ import static org.testng.Assert.assertTrue; import org.apache.fory.ForyTestBase; -import org.apache.fory.memory.NativeByteOrder; -import org.apache.fory.platform.UnsafeOps; import org.apache.fory.serializer.StringEncodingUtils; import org.testng.annotations.Test; public class StringUtilsTest extends ForyTestBase { - @Test public void testEncodeHexString() { assertEquals( @@ -106,50 +103,12 @@ public void testVectorizedLatinCheckAlgorithm(boolean endian) { } private boolean isLatin(char[] chars, boolean isLittle) { - boolean reverseBytes = - (NativeByteOrder.IS_LITTLE_ENDIAN && !isLittle) - || (!NativeByteOrder.IS_LITTLE_ENDIAN && !isLittle); - if (reverseBytes) { - for (int i = 0; i < chars.length; i++) { - chars[i] = Character.reverseBytes(chars[i]); - } - } - long mask; - if (isLittle) { - // latin chars will be 0xXX,0x00;0xXX,0x00 in byte order; - // Using 0x00,0xff(0xff00) to clear latin bits. - mask = 0xff00ff00ff00ff00L; - } else { - // latin chars will be 0x00,0xXX;0x00,0xXX in byte order; - // Using 0x00,0xff(0x00ff) to clear latin bits. - mask = 0x00ff00ff00ff00ffL; - } - int numChars = chars.length; - int vectorizedLen = numChars >> 2; - int vectorizedChars = vectorizedLen << 2; - int endOffset = UnsafeOps.CHAR_ARRAY_OFFSET + (vectorizedChars << 1); - boolean isLatin = true; - for (int offset = UnsafeOps.CHAR_ARRAY_OFFSET; offset < endOffset; offset += 8) { - // check 4 chars in a vectorized way, 4 times faster than scalar check loop. - long multiChars = UnsafeOps.getLong(chars, offset); - if ((multiChars & mask) != 0) { - isLatin = false; - break; - } - } - if (isLatin) { - for (int i = vectorizedChars; i < numChars; i++) { - char c = chars[i]; - if (reverseBytes) { - c = Character.reverseBytes(c); - } - if (c > 0xFF) { - isLatin = false; - break; - } + for (char c : chars) { + if (c > 0xFF) { + return false; } } - return isLatin; + return true; } @Test diff --git a/java/fory-format/pom.xml b/java/fory-format/pom.xml index 15deca64b3..1e9c2adaeb 100644 --- a/java/fory-format/pom.xml +++ b/java/fory-format/pom.xml @@ -116,7 +116,7 @@ maven-surefire-plugin - --add-opens=java.base/java.nio=ALL-UNNAMED + ${argLine} --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED diff --git a/java/fory-format/src/main/java/org/apache/fory/format/row/binary/BinaryArray.java b/java/fory-format/src/main/java/org/apache/fory/format/row/binary/BinaryArray.java index 093c61f71f..8273704f76 100644 --- a/java/fory-format/src/main/java/org/apache/fory/format/row/binary/BinaryArray.java +++ b/java/fory-format/src/main/java/org/apache/fory/format/row/binary/BinaryArray.java @@ -34,7 +34,7 @@ import org.apache.fory.memory.BitUtils; import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.memory.MemoryUtils; -import org.apache.fory.platform.UnsafeOps; +import org.apache.fory.memory.NativeByteOrder; import org.apache.fory.util.Preconditions; /** @@ -47,6 +47,10 @@ *

    Primitive type is always considered to be not null. */ public class BinaryArray extends UnsafeTrait implements ArrayData { + // Row-format stores multi-byte primitive values in little-endian order. MemoryBuffer typed + // array copies are native/raw copies, so big-endian runtimes must use element accessors. + private static final boolean LITTLE_ENDIAN = NativeByteOrder.IS_LITTLE_ENDIAN; + private final Field field; protected final int elementSize; private MemoryBuffer buffer; @@ -199,31 +203,61 @@ public byte[] toByteArray() { public short[] toShortArray() { short[] values = new short[numElements]; - buffer.copyToShortArray(elementOffset, values, 0, numElements * 2); + if (LITTLE_ENDIAN) { + buffer.copyToShortArray(elementOffset, values, 0, numElements * 2); + } else { + for (int i = 0, offset = elementOffset; i < numElements; i++, offset += 2) { + values[i] = buffer.getInt16(offset); + } + } return values; } public int[] toIntArray() { int[] values = new int[numElements]; - buffer.copyToIntArray(elementOffset, values, 0, numElements * 4); + if (LITTLE_ENDIAN) { + buffer.copyToIntArray(elementOffset, values, 0, numElements * 4); + } else { + for (int i = 0, offset = elementOffset; i < numElements; i++, offset += 4) { + values[i] = buffer.getInt32(offset); + } + } return values; } public long[] toLongArray() { long[] values = new long[numElements]; - buffer.copyToLongArray(elementOffset, values, 0, numElements * 8); + if (LITTLE_ENDIAN) { + buffer.copyToLongArray(elementOffset, values, 0, numElements * 8); + } else { + for (int i = 0, offset = elementOffset; i < numElements; i++, offset += 8) { + values[i] = buffer.getInt64(offset); + } + } return values; } public float[] toFloatArray() { float[] values = new float[numElements]; - buffer.copyToFloatArray(elementOffset, values, 0, numElements * 4); + if (LITTLE_ENDIAN) { + buffer.copyToFloatArray(elementOffset, values, 0, numElements * 4); + } else { + for (int i = 0, offset = elementOffset; i < numElements; i++, offset += 4) { + values[i] = buffer.getFloat32(offset); + } + } return values; } public double[] toDoubleArray() { double[] values = new double[numElements]; - buffer.copyToDoubleArray(elementOffset, values, 0, numElements * 8); + if (LITTLE_ENDIAN) { + buffer.copyToDoubleArray(elementOffset, values, 0, numElements * 8); + } else { + for (int i = 0, offset = elementOffset; i < numElements; i++, offset += 8) { + values[i] = buffer.getFloat64(offset); + } + } return values; } @@ -252,7 +286,7 @@ public String toString() { return builder.toString(); } - private static BinaryArray fromPrimitiveArray(Object arr, int offset, int length, Field field) { + private static BinaryArray newPrimitiveArray(int length, Field field) { BinaryArray result = new BinaryArray(field); final long headerInBytes = calculateHeaderInBytes(length); final long valueRegionInBytes = result.elementSize * length; @@ -263,48 +297,82 @@ private static BinaryArray fromPrimitiveArray(Object arr, int offset, int length } final byte[] data = new byte[(int) totalSize]; - UnsafeOps.putLong(data, UnsafeOps.BYTE_ARRAY_OFFSET, length); - UnsafeOps.copyMemory( - arr, offset, data, UnsafeOps.BYTE_ARRAY_OFFSET + headerInBytes, valueRegionInBytes); - MemoryBuffer memoryBuffer = MemoryUtils.wrap(data); + memoryBuffer.putInt64(0, length); result.pointTo(memoryBuffer, 0, (int) totalSize); return result; } public static BinaryArray fromPrimitiveArray(byte[] arr) { - return fromPrimitiveArray( - arr, UnsafeOps.BYTE_ARRAY_OFFSET, arr.length, PRIMITIVE_BYTE_ARRAY_FIELD); + BinaryArray result = newPrimitiveArray(arr.length, PRIMITIVE_BYTE_ARRAY_FIELD); + result.buffer.copyFromByteArray(result.elementOffset, arr, 0, arr.length); + return result; } public static BinaryArray fromPrimitiveArray(boolean[] arr) { - return fromPrimitiveArray( - arr, UnsafeOps.BOOLEAN_ARRAY_OFFSET, arr.length, PRIMITIVE_BOOLEAN_ARRAY_FIELD); + BinaryArray result = newPrimitiveArray(arr.length, PRIMITIVE_BOOLEAN_ARRAY_FIELD); + result.buffer.copyFromBooleanArray(result.elementOffset, arr, 0, arr.length); + return result; } public static BinaryArray fromPrimitiveArray(short[] arr) { - return fromPrimitiveArray( - arr, UnsafeOps.SHORT_ARRAY_OFFSET, arr.length, PRIMITIVE_SHORT_ARRAY_FIELD); + BinaryArray result = newPrimitiveArray(arr.length, PRIMITIVE_SHORT_ARRAY_FIELD); + if (LITTLE_ENDIAN) { + result.buffer.copyFromShortArray(result.elementOffset, arr, 0, arr.length * 2); + } else { + for (int i = 0, offset = result.elementOffset; i < arr.length; i++, offset += 2) { + result.buffer.putInt16(offset, arr[i]); + } + } + return result; } public static BinaryArray fromPrimitiveArray(int[] arr) { - return fromPrimitiveArray( - arr, UnsafeOps.INT_ARRAY_OFFSET, arr.length, PRIMITIVE_INT_ARRAY_FIELD); + BinaryArray result = newPrimitiveArray(arr.length, PRIMITIVE_INT_ARRAY_FIELD); + if (LITTLE_ENDIAN) { + result.buffer.copyFromIntArray(result.elementOffset, arr, 0, arr.length * 4); + } else { + for (int i = 0, offset = result.elementOffset; i < arr.length; i++, offset += 4) { + result.buffer.putInt32(offset, arr[i]); + } + } + return result; } public static BinaryArray fromPrimitiveArray(long[] arr) { - return fromPrimitiveArray( - arr, UnsafeOps.LONG_ARRAY_OFFSET, arr.length, PRIMITIVE_LONG_ARRAY_FIELD); + BinaryArray result = newPrimitiveArray(arr.length, PRIMITIVE_LONG_ARRAY_FIELD); + if (LITTLE_ENDIAN) { + result.buffer.copyFromLongArray(result.elementOffset, arr, 0, arr.length * 8); + } else { + for (int i = 0, offset = result.elementOffset; i < arr.length; i++, offset += 8) { + result.buffer.putInt64(offset, arr[i]); + } + } + return result; } public static BinaryArray fromPrimitiveArray(float[] arr) { - return fromPrimitiveArray( - arr, UnsafeOps.FLOAT_ARRAY_OFFSET, arr.length, PRIMITIVE_FLOAT_ARRAY_FIELD); + BinaryArray result = newPrimitiveArray(arr.length, PRIMITIVE_FLOAT_ARRAY_FIELD); + if (LITTLE_ENDIAN) { + result.buffer.copyFromFloatArray(result.elementOffset, arr, 0, arr.length * 4); + } else { + for (int i = 0, offset = result.elementOffset; i < arr.length; i++, offset += 4) { + result.buffer.putFloat32(offset, arr[i]); + } + } + return result; } public static BinaryArray fromPrimitiveArray(double[] arr) { - return fromPrimitiveArray( - arr, UnsafeOps.DOUBLE_ARRAY_OFFSET, arr.length, PRIMITIVE_DOUBLE_ARRAY_FIELD); + BinaryArray result = newPrimitiveArray(arr.length, PRIMITIVE_DOUBLE_ARRAY_FIELD); + if (LITTLE_ENDIAN) { + result.buffer.copyFromDoubleArray(result.elementOffset, arr, 0, arr.length * 8); + } else { + for (int i = 0, offset = result.elementOffset; i < arr.length; i++, offset += 8) { + result.buffer.putFloat64(offset, arr[i]); + } + } + return result; } public static int calculateHeaderInBytes(int numElements) { diff --git a/java/fory-format/src/main/java/org/apache/fory/format/row/binary/writer/BinaryArrayWriter.java b/java/fory-format/src/main/java/org/apache/fory/format/row/binary/writer/BinaryArrayWriter.java index ed656b6329..7aff4a387e 100644 --- a/java/fory-format/src/main/java/org/apache/fory/format/row/binary/writer/BinaryArrayWriter.java +++ b/java/fory-format/src/main/java/org/apache/fory/format/row/binary/writer/BinaryArrayWriter.java @@ -33,7 +33,7 @@ import org.apache.fory.format.type.Field; import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.memory.MemoryUtils; -import org.apache.fory.platform.UnsafeOps; +import org.apache.fory.memory.NativeByteOrder; /** * Writer for binary array. See {@link BinaryArray} @@ -45,6 +45,8 @@ * fromPrimitiveArray. */ public class BinaryArrayWriter extends BinaryWriter { + private static final boolean LITTLE_ENDIAN = NativeByteOrder.IS_LITTLE_ENDIAN; + public static int MAX_ROUNDED_ARRAY_LENGTH = Integer.MAX_VALUE - 15; protected final Field field; @@ -184,7 +186,7 @@ protected void primitiveArrayAdvance(int size) { // no need to increasewriterIndex, because reset has already increased writerIndex } - private void fromPrimitiveArray(Object arr, int offset, int numElements, Field type) { + private void checkPrimitiveArrayType(Field type) { DataTypes.ListType inputListType = (DataTypes.ListType) type.type(); DataTypes.ListType thisListType = (DataTypes.ListType) this.field.type(); if (DataTypes.getTypeId(inputListType.valueType()) @@ -194,39 +196,88 @@ private void fromPrimitiveArray(Object arr, int offset, int numElements, Field t "Element type %s is not %s", inputListType.valueType(), thisListType.valueType()); throw new IllegalArgumentException(msg); } + } + + private void finishPrimitiveArray(int numElements) { int size = numElements * elementSize; - buffer.copyFromUnsafe(startIndex + headerInBytes, arr, offset, size); primitiveArrayAdvance(size); } public void fromPrimitiveArray(byte[] arr) { - fromPrimitiveArray(arr, UnsafeOps.BYTE_ARRAY_OFFSET, arr.length, PRIMITIVE_BYTE_ARRAY_FIELD); + checkPrimitiveArrayType(PRIMITIVE_BYTE_ARRAY_FIELD); + buffer.copyFromByteArray(startIndex + headerInBytes, arr, 0, arr.length); + finishPrimitiveArray(arr.length); } public void fromPrimitiveArray(boolean[] arr) { - fromPrimitiveArray( - arr, UnsafeOps.BOOLEAN_ARRAY_OFFSET, arr.length, PRIMITIVE_BOOLEAN_ARRAY_FIELD); + checkPrimitiveArrayType(PRIMITIVE_BOOLEAN_ARRAY_FIELD); + buffer.copyFromBooleanArray(startIndex + headerInBytes, arr, 0, arr.length); + finishPrimitiveArray(arr.length); } public void fromPrimitiveArray(short[] arr) { - fromPrimitiveArray(arr, UnsafeOps.SHORT_ARRAY_OFFSET, arr.length, PRIMITIVE_SHORT_ARRAY_FIELD); + checkPrimitiveArrayType(PRIMITIVE_SHORT_ARRAY_FIELD); + int offset = startIndex + headerInBytes; + if (LITTLE_ENDIAN) { + buffer.copyFromShortArray(offset, arr, 0, arr.length * 2); + } else { + for (int i = 0; i < arr.length; i++, offset += 2) { + buffer.putInt16(offset, arr[i]); + } + } + finishPrimitiveArray(arr.length); } public void fromPrimitiveArray(int[] arr) { - fromPrimitiveArray(arr, UnsafeOps.INT_ARRAY_OFFSET, arr.length, PRIMITIVE_INT_ARRAY_FIELD); + checkPrimitiveArrayType(PRIMITIVE_INT_ARRAY_FIELD); + int offset = startIndex + headerInBytes; + if (LITTLE_ENDIAN) { + buffer.copyFromIntArray(offset, arr, 0, arr.length * 4); + } else { + for (int i = 0; i < arr.length; i++, offset += 4) { + buffer.putInt32(offset, arr[i]); + } + } + finishPrimitiveArray(arr.length); } public void fromPrimitiveArray(long[] arr) { - fromPrimitiveArray(arr, UnsafeOps.LONG_ARRAY_OFFSET, arr.length, PRIMITIVE_LONG_ARRAY_FIELD); + checkPrimitiveArrayType(PRIMITIVE_LONG_ARRAY_FIELD); + int offset = startIndex + headerInBytes; + if (LITTLE_ENDIAN) { + buffer.copyFromLongArray(offset, arr, 0, arr.length * 8); + } else { + for (int i = 0; i < arr.length; i++, offset += 8) { + buffer.putInt64(offset, arr[i]); + } + } + finishPrimitiveArray(arr.length); } public void fromPrimitiveArray(float[] arr) { - fromPrimitiveArray(arr, UnsafeOps.FLOAT_ARRAY_OFFSET, arr.length, PRIMITIVE_FLOAT_ARRAY_FIELD); + checkPrimitiveArrayType(PRIMITIVE_FLOAT_ARRAY_FIELD); + int offset = startIndex + headerInBytes; + if (LITTLE_ENDIAN) { + buffer.copyFromFloatArray(offset, arr, 0, arr.length * 4); + } else { + for (int i = 0; i < arr.length; i++, offset += 4) { + buffer.putFloat32(offset, arr[i]); + } + } + finishPrimitiveArray(arr.length); } public void fromPrimitiveArray(double[] arr) { - fromPrimitiveArray( - arr, UnsafeOps.DOUBLE_ARRAY_OFFSET, arr.length, PRIMITIVE_DOUBLE_ARRAY_FIELD); + checkPrimitiveArrayType(PRIMITIVE_DOUBLE_ARRAY_FIELD); + int offset = startIndex + headerInBytes; + if (LITTLE_ENDIAN) { + buffer.copyFromDoubleArray(offset, arr, 0, arr.length * 8); + } else { + for (int i = 0; i < arr.length; i++, offset += 8) { + buffer.putFloat64(offset, arr[i]); + } + } + finishPrimitiveArray(arr.length); } public BinaryArray toArray() { diff --git a/java/fory-format/src/test/java/org/apache/fory/format/row/binary/BinaryArrayTest.java b/java/fory-format/src/test/java/org/apache/fory/format/row/binary/BinaryArrayTest.java index 06aaa9d262..2e73dce860 100644 --- a/java/fory-format/src/test/java/org/apache/fory/format/row/binary/BinaryArrayTest.java +++ b/java/fory-format/src/test/java/org/apache/fory/format/row/binary/BinaryArrayTest.java @@ -19,6 +19,7 @@ package org.apache.fory.format.row.binary; +import java.util.Arrays; import java.util.Random; import org.apache.fory.format.row.binary.writer.BinaryArrayWriter; import org.apache.fory.format.type.DataTypes; @@ -40,6 +41,109 @@ public void fromPrimitiveArray() { writer.toArray(); } + @Test + public void primitiveArrayRoundTrip() { + assertRoundTrip(new byte[] {1, -2, 3}); + assertRoundTrip(new boolean[] {true, false, true}); + assertRoundTrip(new short[] {1, -2, Short.MAX_VALUE}); + assertRoundTrip(new int[] {1, -2, Integer.MAX_VALUE}); + assertRoundTrip(new long[] {1L, -2L, Long.MAX_VALUE}); + assertRoundTrip(new float[] {1.25f, -2.5f, Float.MAX_VALUE}); + assertRoundTrip(new double[] {1.25d, -2.5d, Double.MAX_VALUE}); + } + + @Test + public void primitiveWireEndian() { + assertValueBytes( + BinaryArray.fromPrimitiveArray(new short[] {(short) 0x1234}), bytes(0x34, 0x12)); + assertValueBytes( + BinaryArray.fromPrimitiveArray(new int[] {0x12345678}), bytes(0x78, 0x56, 0x34, 0x12)); + assertValueBytes( + BinaryArray.fromPrimitiveArray(new long[] {0x0102030405060708L}), + bytes(0x08, 0x07, 0x06, 0x05, 0x04, 0x03, 0x02, 0x01)); + assertValueBytes( + BinaryArray.fromPrimitiveArray(new float[] {Float.intBitsToFloat(0x12345678)}), + bytes(0x78, 0x56, 0x34, 0x12)); + assertValueBytes( + BinaryArray.fromPrimitiveArray(new double[] {Double.longBitsToDouble(0x0102030405060708L)}), + bytes(0x08, 0x07, 0x06, 0x05, 0x04, 0x03, 0x02, 0x01)); + + BinaryArrayWriter writer = new BinaryArrayWriter(DataTypes.PRIMITIVE_INT_ARRAY_FIELD); + writer.reset(1); + writer.fromPrimitiveArray(new int[] {0x12345678}); + assertValueBytes(writer.toArray(), bytes(0x78, 0x56, 0x34, 0x12)); + } + + private static void assertRoundTrip(byte[] arr) { + Assert.assertEquals(BinaryArray.fromPrimitiveArray(arr).toByteArray(), arr); + BinaryArrayWriter writer = new BinaryArrayWriter(DataTypes.PRIMITIVE_BYTE_ARRAY_FIELD); + writer.reset(arr.length); + writer.fromPrimitiveArray(arr); + Assert.assertEquals(writer.toArray().toByteArray(), arr); + } + + private static void assertRoundTrip(boolean[] arr) { + Assert.assertEquals(BinaryArray.fromPrimitiveArray(arr).toBooleanArray(), arr); + BinaryArrayWriter writer = new BinaryArrayWriter(DataTypes.PRIMITIVE_BOOLEAN_ARRAY_FIELD); + writer.reset(arr.length); + writer.fromPrimitiveArray(arr); + Assert.assertEquals(writer.toArray().toBooleanArray(), arr); + } + + private static void assertRoundTrip(short[] arr) { + Assert.assertEquals(BinaryArray.fromPrimitiveArray(arr).toShortArray(), arr); + BinaryArrayWriter writer = new BinaryArrayWriter(DataTypes.PRIMITIVE_SHORT_ARRAY_FIELD); + writer.reset(arr.length); + writer.fromPrimitiveArray(arr); + Assert.assertEquals(writer.toArray().toShortArray(), arr); + } + + private static void assertRoundTrip(int[] arr) { + Assert.assertEquals(BinaryArray.fromPrimitiveArray(arr).toIntArray(), arr); + BinaryArrayWriter writer = new BinaryArrayWriter(DataTypes.PRIMITIVE_INT_ARRAY_FIELD); + writer.reset(arr.length); + writer.fromPrimitiveArray(arr); + Assert.assertEquals(writer.toArray().toIntArray(), arr); + } + + private static void assertRoundTrip(long[] arr) { + Assert.assertEquals(BinaryArray.fromPrimitiveArray(arr).toLongArray(), arr); + BinaryArrayWriter writer = new BinaryArrayWriter(DataTypes.PRIMITIVE_LONG_ARRAY_FIELD); + writer.reset(arr.length); + writer.fromPrimitiveArray(arr); + Assert.assertEquals(writer.toArray().toLongArray(), arr); + } + + private static void assertRoundTrip(float[] arr) { + Assert.assertEquals(BinaryArray.fromPrimitiveArray(arr).toFloatArray(), arr); + BinaryArrayWriter writer = new BinaryArrayWriter(DataTypes.PRIMITIVE_FLOAT_ARRAY_FIELD); + writer.reset(arr.length); + writer.fromPrimitiveArray(arr); + Assert.assertEquals(writer.toArray().toFloatArray(), arr); + } + + private static void assertRoundTrip(double[] arr) { + Assert.assertEquals(BinaryArray.fromPrimitiveArray(arr).toDoubleArray(), arr); + BinaryArrayWriter writer = new BinaryArrayWriter(DataTypes.PRIMITIVE_DOUBLE_ARRAY_FIELD); + writer.reset(arr.length); + writer.fromPrimitiveArray(arr); + Assert.assertEquals(writer.toArray().toDoubleArray(), arr); + } + + private static void assertValueBytes(BinaryArray array, byte[] expected) { + byte[] bytes = array.toBytes(); + int offset = BinaryArray.calculateHeaderInBytes(1); + Assert.assertEquals(Arrays.copyOfRange(bytes, offset, offset + expected.length), expected); + } + + private static byte[] bytes(int... values) { + byte[] bytes = new byte[values.length]; + for (int i = 0; i < values.length; i++) { + bytes[i] = (byte) values[i]; + } + return bytes; + } + private int elem; @Test(enabled = false) diff --git a/java/fory-format/src/test/java/org/apache/fory/format/row/binary/BinaryRowTest.java b/java/fory-format/src/test/java/org/apache/fory/format/row/binary/BinaryRowTest.java index 803ce833f5..a68da3bb23 100644 --- a/java/fory-format/src/test/java/org/apache/fory/format/row/binary/BinaryRowTest.java +++ b/java/fory-format/src/test/java/org/apache/fory/format/row/binary/BinaryRowTest.java @@ -31,7 +31,6 @@ import org.apache.fory.logging.LoggerFactory; import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.memory.MemoryUtils; -import org.apache.fory.platform.UnsafeOps; import org.testng.annotations.Test; public class BinaryRowTest { @@ -86,13 +85,14 @@ public void testOffsetAccessPerf() { offsetsArray[i] = i; } - byte[] bytes = new byte[numFields]; + int headerInBytes = 64; + byte[] bytes = new byte[headerInBytes + 8 * numFields]; int iterNums = 1000_000_000; // warm for (int i = 0; i < iterNums; i++) { for (int j = 0; j < numFields; j++) { int tmp = offsetsArray[j]; - UnsafeOps.getByte(bytes, tmp); + byte ignored = bytes[tmp]; } } // test access offset array @@ -100,18 +100,17 @@ public void testOffsetAccessPerf() { for (int i = 0; i < iterNums; i++) { for (int j = 0; j < numFields; j++) { int tmp = offsetsArray[j]; - UnsafeOps.getByte(bytes, tmp); + byte ignored = bytes[tmp]; } } long duration = System.nanoTime() - startTime; LOG.info("Array access offset take " + duration + "ns, " + duration / 1000_000 + " ms\n"); - int headerInBytes = 64; // warm for (int i = 0; i < iterNums; i++) { for (int j = 0; j < numFields; j++) { int tmp = headerInBytes + 8 * j; - UnsafeOps.getByte(bytes, tmp); + byte ignored = bytes[tmp]; } } // test calc offset @@ -119,7 +118,7 @@ public void testOffsetAccessPerf() { for (int i = 0; i < iterNums; i++) { for (int j = 0; j < numFields; j++) { int tmp = headerInBytes + 8 * j; - UnsafeOps.getByte(bytes, tmp); + byte ignored = bytes[tmp]; } } duration = System.nanoTime() - startTime; diff --git a/java/fory-simd/pom.xml b/java/fory-simd/pom.xml index 675e183748..0202698c3c 100644 --- a/java/fory-simd/pom.xml +++ b/java/fory-simd/pom.xml @@ -74,7 +74,7 @@ org.apache.maven.plugins maven-surefire-plugin - --add-modules=jdk.incubator.vector + ${argLine} --add-modules=jdk.incubator.vector diff --git a/java/pom.xml b/java/pom.xml index 7884abb1bc..8d35a37597 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -74,6 +74,7 @@ 3.1.12 1.13 ${basedir} + 3.3.0 1.18.38 4.11.0 diff --git a/kotlin/fory-kotlin/pom.xml b/kotlin/fory-kotlin/pom.xml index 139cc6dc7e..00719b4789 100644 --- a/kotlin/fory-kotlin/pom.xml +++ b/kotlin/fory-kotlin/pom.xml @@ -44,6 +44,7 @@ compile + true ${project.basedir}/src/main/kotlin ${project.basedir}/src/main/java @@ -60,6 +61,7 @@ test-compile + true ${project.basedir}/src/test/kotlin diff --git a/kotlin/fory-kotlin/src/main/java/org/apache/fory/serializer/kotlin/KotlinSerializers.java b/kotlin/fory-kotlin/src/main/java/org/apache/fory/serializer/kotlin/KotlinSerializers.java index 36acb0c5d8..c830124196 100644 --- a/kotlin/fory-kotlin/src/main/java/org/apache/fory/serializer/kotlin/KotlinSerializers.java +++ b/kotlin/fory-kotlin/src/main/java/org/apache/fory/serializer/kotlin/KotlinSerializers.java @@ -125,20 +125,38 @@ public static void registerSerializers(Fory fory) { // Ranges and Progressions. registerIfAbsent(resolver, kotlin.ranges.CharRange.class); + resolver.registerSerializer(kotlin.ranges.CharRange.class, new CharRangeSerializer(config)); registerIfAbsent(resolver, kotlin.ranges.CharProgression.class); + resolver.registerSerializer( + kotlin.ranges.CharProgression.class, new CharProgressionSerializer(config)); registerIfAbsent(resolver, kotlin.ranges.IntRange.class); + resolver.registerSerializer(kotlin.ranges.IntRange.class, new IntRangeSerializer(config)); registerIfAbsent(resolver, kotlin.ranges.IntProgression.class); + resolver.registerSerializer( + kotlin.ranges.IntProgression.class, new IntProgressionSerializer(config)); registerIfAbsent(resolver, kotlin.ranges.LongRange.class); + resolver.registerSerializer(kotlin.ranges.LongRange.class, new LongRangeSerializer(config)); registerIfAbsent(resolver, kotlin.ranges.LongProgression.class); + resolver.registerSerializer( + kotlin.ranges.LongProgression.class, new LongProgressionSerializer(config)); registerIfAbsent(resolver, kotlin.ranges.UIntRange.class); + resolver.registerSerializer(kotlin.ranges.UIntRange.class, new UIntRangeSerializer(config)); registerIfAbsent(resolver, kotlin.ranges.UIntProgression.class); + resolver.registerSerializer( + kotlin.ranges.UIntProgression.class, new UIntProgressionSerializer(config)); registerIfAbsent(resolver, kotlin.ranges.ULongRange.class); + resolver.registerSerializer(kotlin.ranges.ULongRange.class, new ULongRangeSerializer(config)); registerIfAbsent(resolver, kotlin.ranges.ULongProgression.class); + resolver.registerSerializer( + kotlin.ranges.ULongProgression.class, new ULongProgressionSerializer(config)); // Built-in classes. registerIfAbsent(resolver, kotlin.Pair.class); + resolver.registerSerializer(kotlin.Pair.class, new PairSerializer(config)); registerIfAbsent(resolver, kotlin.Triple.class); + resolver.registerSerializer(kotlin.Triple.class, new TripleSerializer(config)); registerIfAbsent(resolver, kotlin.Result.class); + resolver.registerSerializer(kotlin.Result.class, new ResultSerializer(config)); registerIfAbsent(resolver, Result.Failure.class); // kotlin.random diff --git a/kotlin/fory-kotlin/src/main/kotlin/org/apache/fory/serializer/kotlin/KotlinBuiltinSerializers.kt b/kotlin/fory-kotlin/src/main/kotlin/org/apache/fory/serializer/kotlin/KotlinBuiltinSerializers.kt new file mode 100644 index 0000000000..56362b8525 --- /dev/null +++ b/kotlin/fory-kotlin/src/main/kotlin/org/apache/fory/serializer/kotlin/KotlinBuiltinSerializers.kt @@ -0,0 +1,236 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.serializer.kotlin + +import org.apache.fory.config.Config +import org.apache.fory.context.ReadContext +import org.apache.fory.context.WriteContext +import org.apache.fory.serializer.ImmutableSerializer +import org.apache.fory.serializer.Shareable + +public class PairSerializer(config: Config) : + ImmutableSerializer>(config, Pair::class.java), Shareable { + override fun write(writeContext: WriteContext, value: Pair<*, *>) { + writeContext.writeRef(value.first) + writeContext.writeRef(value.second) + } + + override fun read(readContext: ReadContext): Pair<*, *> { + return Pair(readContext.readRef(), readContext.readRef()) + } +} + +public class TripleSerializer(config: Config) : + ImmutableSerializer>(config, Triple::class.java), Shareable { + override fun write(writeContext: WriteContext, value: Triple<*, *, *>) { + writeContext.writeRef(value.first) + writeContext.writeRef(value.second) + writeContext.writeRef(value.third) + } + + override fun read(readContext: ReadContext): Triple<*, *, *> { + return Triple(readContext.readRef(), readContext.readRef(), readContext.readRef()) + } +} + +public class ResultSerializer(config: Config) : + ImmutableSerializer>(config, Result::class.java), Shareable { + override fun write(writeContext: WriteContext, value: Result<*>) { + val failure = value.exceptionOrNull() + val buffer = writeContext.buffer + buffer.writeBoolean(failure == null) + if (failure == null) { + writeContext.writeRef(value.getOrNull()) + } else { + writeContext.writeRef(failure) + } + } + + override fun read(readContext: ReadContext): Result<*> { + return if (readContext.buffer.readBoolean()) { + Result.success(readContext.readRef()) + } else { + Result.failure(readContext.readRef() as Throwable) + } + } +} + +public class CharRangeSerializer(config: Config) : + ImmutableSerializer(config, CharRange::class.java), Shareable { + override fun write(writeContext: WriteContext, value: CharRange) { + val buffer = writeContext.buffer + buffer.writeInt16(value.first.code.toShort()) + buffer.writeInt16(value.last.code.toShort()) + } + + override fun read(readContext: ReadContext): CharRange { + val buffer = readContext.buffer + return CharRange(buffer.readInt16().toInt().toChar(), buffer.readInt16().toInt().toChar()) + } +} + +public class CharProgressionSerializer(config: Config) : + ImmutableSerializer(config, CharProgression::class.java), Shareable { + override fun write(writeContext: WriteContext, value: CharProgression) { + val buffer = writeContext.buffer + buffer.writeInt16(value.first.code.toShort()) + buffer.writeInt16(value.last.code.toShort()) + buffer.writeInt32(value.step) + } + + override fun read(readContext: ReadContext): CharProgression { + val buffer = readContext.buffer + return CharProgression.fromClosedRange( + buffer.readInt16().toInt().toChar(), + buffer.readInt16().toInt().toChar(), + buffer.readInt32() + ) + } +} + +public class IntRangeSerializer(config: Config) : + ImmutableSerializer(config, IntRange::class.java), Shareable { + override fun write(writeContext: WriteContext, value: IntRange) { + val buffer = writeContext.buffer + buffer.writeInt32(value.first) + buffer.writeInt32(value.last) + } + + override fun read(readContext: ReadContext): IntRange { + val buffer = readContext.buffer + return IntRange(buffer.readInt32(), buffer.readInt32()) + } +} + +public class IntProgressionSerializer(config: Config) : + ImmutableSerializer(config, IntProgression::class.java), Shareable { + override fun write(writeContext: WriteContext, value: IntProgression) { + val buffer = writeContext.buffer + buffer.writeInt32(value.first) + buffer.writeInt32(value.last) + buffer.writeInt32(value.step) + } + + override fun read(readContext: ReadContext): IntProgression { + val buffer = readContext.buffer + return IntProgression.fromClosedRange( + buffer.readInt32(), + buffer.readInt32(), + buffer.readInt32() + ) + } +} + +public class LongRangeSerializer(config: Config) : + ImmutableSerializer(config, LongRange::class.java), Shareable { + override fun write(writeContext: WriteContext, value: LongRange) { + val buffer = writeContext.buffer + buffer.writeInt64(value.first) + buffer.writeInt64(value.last) + } + + override fun read(readContext: ReadContext): LongRange { + val buffer = readContext.buffer + return LongRange(buffer.readInt64(), buffer.readInt64()) + } +} + +public class LongProgressionSerializer(config: Config) : + ImmutableSerializer(config, LongProgression::class.java), Shareable { + override fun write(writeContext: WriteContext, value: LongProgression) { + val buffer = writeContext.buffer + buffer.writeInt64(value.first) + buffer.writeInt64(value.last) + buffer.writeInt64(value.step) + } + + override fun read(readContext: ReadContext): LongProgression { + val buffer = readContext.buffer + val first = buffer.readInt64() + val last = buffer.readInt64() + val step = buffer.readInt64() + return LongProgression.fromClosedRange(first, last, step) + } +} + +public class UIntRangeSerializer(config: Config) : + ImmutableSerializer(config, UIntRange::class.java), Shareable { + override fun write(writeContext: WriteContext, value: UIntRange) { + val buffer = writeContext.buffer + buffer.writeInt32(value.first.toInt()) + buffer.writeInt32(value.last.toInt()) + } + + override fun read(readContext: ReadContext): UIntRange { + val buffer = readContext.buffer + return UIntRange(buffer.readInt32().toUInt(), buffer.readInt32().toUInt()) + } +} + +public class UIntProgressionSerializer(config: Config) : + ImmutableSerializer(config, UIntProgression::class.java), Shareable { + override fun write(writeContext: WriteContext, value: UIntProgression) { + val buffer = writeContext.buffer + buffer.writeInt32(value.first.toInt()) + buffer.writeInt32(value.last.toInt()) + buffer.writeInt32(value.step) + } + + override fun read(readContext: ReadContext): UIntProgression { + val buffer = readContext.buffer + return UIntProgression.fromClosedRange( + buffer.readInt32().toUInt(), + buffer.readInt32().toUInt(), + buffer.readInt32() + ) + } +} + +public class ULongRangeSerializer(config: Config) : + ImmutableSerializer(config, ULongRange::class.java), Shareable { + override fun write(writeContext: WriteContext, value: ULongRange) { + val buffer = writeContext.buffer + buffer.writeInt64(value.first.toLong()) + buffer.writeInt64(value.last.toLong()) + } + + override fun read(readContext: ReadContext): ULongRange { + val buffer = readContext.buffer + return ULongRange(buffer.readInt64().toULong(), buffer.readInt64().toULong()) + } +} + +public class ULongProgressionSerializer(config: Config) : + ImmutableSerializer(config, ULongProgression::class.java), Shareable { + override fun write(writeContext: WriteContext, value: ULongProgression) { + val buffer = writeContext.buffer + buffer.writeInt64(value.first.toLong()) + buffer.writeInt64(value.last.toLong()) + buffer.writeInt64(value.step) + } + + override fun read(readContext: ReadContext): ULongProgression { + val buffer = readContext.buffer + val first = buffer.readInt64().toULong() + val last = buffer.readInt64().toULong() + val step = buffer.readInt64() + return ULongProgression.fromClosedRange(first, last, step) + } +} diff --git a/kotlin/fory-kotlin/src/main/kotlin/org/apache/fory/serializer/kotlin/KotlinDefaultValueSupport.kt b/kotlin/fory-kotlin/src/main/kotlin/org/apache/fory/serializer/kotlin/KotlinDefaultValueSupport.kt index aa8caf93b2..26850ff5d6 100644 --- a/kotlin/fory-kotlin/src/main/kotlin/org/apache/fory/serializer/kotlin/KotlinDefaultValueSupport.kt +++ b/kotlin/fory-kotlin/src/main/kotlin/org/apache/fory/serializer/kotlin/KotlinDefaultValueSupport.kt @@ -19,7 +19,10 @@ package org.apache.fory.serializer.kotlin +import java.lang.reflect.Array +import java.lang.reflect.Modifier import java.lang.reflect.Type +import java.util.IdentityHashMap import kotlin.reflect.KParameter import kotlin.reflect.full.memberProperties import kotlin.reflect.full.primaryConstructor @@ -28,7 +31,7 @@ import org.apache.fory.collection.ClassValueCache import org.apache.fory.logging.Logger import org.apache.fory.logging.LoggerFactory import org.apache.fory.platform.AndroidSupport -import org.apache.fory.platform.UnsafeOps +import org.apache.fory.reflect.ObjectCreators import org.apache.fory.util.DefaultValueUtils /** @@ -68,7 +71,7 @@ internal class KotlinDefaultValueSupport : DefaultValueUtils.DefaultValueSupport // Provide default values for all required (non-optional) parameters for (parameter in parameters) { if (!parameter.isOptional) { - val defaultValue = getDefaultValueForType(parameter.type.javaType) + val defaultValue = getDefaultValueForType(parameter.type.javaType, IdentityHashMap()) if (defaultValue != null) { argsMap[parameter] = defaultValue } else { @@ -87,7 +90,7 @@ internal class KotlinDefaultValueSupport : DefaultValueUtils.DefaultValueSupport val property = kClass.memberProperties.find { it.name == parameter.name } property?.let { prop -> @Suppress("UNCHECKED_CAST") - val value = (prop as kotlin.reflect.KProperty1).get(instance as Any) + val value = (prop as kotlin.reflect.KProperty1).get(instance) if (value != null) { defaultValues[parameter.name!!] = value } @@ -114,7 +117,7 @@ internal class KotlinDefaultValueSupport : DefaultValueUtils.DefaultValueSupport return cls.kotlin.isData } - private fun getDefaultValueForType(type: Type): Any? { + private fun getDefaultValueForType(type: Type, seen: IdentityHashMap, Boolean>): Any? { val clazz = when (type) { is Class<*> -> type @@ -140,7 +143,68 @@ internal class KotlinDefaultValueSupport : DefaultValueUtils.DefaultValueSupport Char::class.java, Char::class.javaPrimitiveType -> '\u0000' String::class.java -> "" - else -> if (AndroidSupport.IS_ANDROID) null else UnsafeOps.newInstance(clazz) + else -> getDefaultValueForClass(clazz, seen) } } + + private fun getDefaultValueForClass( + clazz: Class<*>, + seen: IdentityHashMap, Boolean> + ): Any? { + if (clazz.isArray) { + return Array.newInstance(clazz.componentType, 0) + } + if (clazz.isEnum) { + return clazz.enumConstants?.firstOrNull() + } + if (List::class.java.isAssignableFrom(clazz)) { + return emptyList() + } + if (Set::class.java.isAssignableFrom(clazz)) { + return emptySet() + } + if (Map::class.java.isAssignableFrom(clazz)) { + return emptyMap() + } + if (clazz.isInterface || Modifier.isAbstract(clazz.modifiers) || seen.containsKey(clazz)) { + return null + } + seen[clazz] = true + try { + if (!AndroidSupport.IS_ANDROID) { + return newDefaultInstance(clazz, seen) + } + return newPublicDefaultInstance(clazz, seen) + } finally { + seen.remove(clazz) + } + } + + private fun newDefaultInstance(clazz: Class<*>, seen: IdentityHashMap, Boolean>): Any? { + val creator = ObjectCreators.getObjectCreator(clazz) + if (!creator.hasConstructorFields()) { + return creator.newInstance() + } + val parameterTypes = creator.getConstructorFieldTypes() + val args = arrayOfNulls(parameterTypes.size) + for (i in parameterTypes.indices) { + args[i] = getDefaultValueForType(parameterTypes[i], seen) ?: return null + } + return creator.newInstanceWithArguments(*args) + } + + private fun newPublicDefaultInstance( + clazz: Class<*>, + seen: IdentityHashMap, Boolean> + ): Any? { + val constructor = + clazz.constructors.minWithOrNull( + compareBy> { it.parameterCount } + ) ?: return null + val args = + constructor.genericParameterTypes.map { parameterType -> + getDefaultValueForType(parameterType, seen) ?: return null + } + return constructor.newInstance(*args.toTypedArray()) + } } diff --git a/kotlin/fory-kotlin/src/test/kotlin/org/apache/fory/serializer/kotlin/DefaultValueTest.kt b/kotlin/fory-kotlin/src/test/kotlin/org/apache/fory/serializer/kotlin/DefaultValueTest.kt index 62fa3ef5e3..67c76a1ab2 100644 --- a/kotlin/fory-kotlin/src/test/kotlin/org/apache/fory/serializer/kotlin/DefaultValueTest.kt +++ b/kotlin/fory-kotlin/src/test/kotlin/org/apache/fory/serializer/kotlin/DefaultValueTest.kt @@ -131,7 +131,12 @@ class DefaultValueTest { @Test fun testDefaultValueDeserialization() { - val fory = ForyKotlin.builder().withXlang(false).requireClassRegistration(false).withCompatible(true).build() + val fory = + ForyKotlin.builder() + .withXlang(false) + .requireClassRegistration(false) + .withCompatible(true) + .build() val obj = ClassNoDefaults("test") val serialized = fory.serialize(obj) val deserialized = fory.deserialize(serialized, ClassWithDefaults::class.java) diff --git a/scala/build.sbt b/scala/build.sbt index a51d0aa55f..228b4357d7 100644 --- a/scala/build.sbt +++ b/scala/build.sbt @@ -47,6 +47,20 @@ libraryDependencies ++= Seq( "dev.zio" %% "zio" % "2.1.7" % Test, ) +Test / fork := true +Test / javaOptions ++= Seq( + "--add-opens=java.base/java.lang=ALL-UNNAMED", + "--add-opens=java.base/java.lang.invoke=ALL-UNNAMED", + "--add-opens=java.base/java.lang.reflect=ALL-UNNAMED", + "--add-opens=java.base/jdk.internal.reflect=ALL-UNNAMED", + "--add-opens=java.base/java.util=ALL-UNNAMED", + "--add-opens=java.base/java.util.concurrent=ALL-UNNAMED", + "--add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED", + "--add-opens=java.base/java.io=ALL-UNNAMED", + "--add-opens=java.base/java.net=ALL-UNNAMED", + "--add-opens=java.base/java.math=ALL-UNNAMED", +) + lazy val writeTestClasspath = taskKey[File]("Writes the Scala test runtime classpath") writeTestClasspath := { diff --git a/scala/src/main/java/org/apache/fory/serializer/scala/ToFactorySerializers.java b/scala/src/main/java/org/apache/fory/serializer/scala/ToFactorySerializers.java index 81c0b7019c..8fddbd8d2c 100644 --- a/scala/src/main/java/org/apache/fory/serializer/scala/ToFactorySerializers.java +++ b/scala/src/main/java/org/apache/fory/serializer/scala/ToFactorySerializers.java @@ -19,31 +19,33 @@ package org.apache.fory.serializer.scala; +import java.lang.reflect.Constructor; +import java.lang.reflect.Field; import org.apache.fory.config.Config; import org.apache.fory.context.ReadContext; import org.apache.fory.context.WriteContext; -import org.apache.fory.memory.MemoryBuffer; -import org.apache.fory.platform.UnsafeOps; +import org.apache.fory.exception.ForyException; +import org.apache.fory.reflect.FieldAccessor; import org.apache.fory.reflect.ReflectionUtils; import org.apache.fory.serializer.Shareable; import org.apache.fory.serializer.Serializer; -import java.lang.reflect.Field; - -public class ToFactorySerializers { - static final Class IterableToFactoryClass = ReflectionUtils.loadClass( - "scala.collection.IterableFactory$ToFactory"); - static final Class MapToFactoryClass = ReflectionUtils.loadClass( - "scala.collection.MapFactory$ToFactory"); +public class ToFactorySerializers { + static final Class IterableToFactoryClass = + ReflectionUtils.loadClass("scala.collection.IterableFactory$ToFactory"); + static final Class MapToFactoryClass = + ReflectionUtils.loadClass("scala.collection.MapFactory$ToFactory"); public static class IterableToFactorySerializer extends Serializer implements Shareable { - private static final long fieldOffset; + private static final FieldAccessor FACTORY_ACCESSOR; + private static final Constructor CONSTRUCTOR; static { try { - // for graalvm field offset auto rewrite - Field field = Class.forName("scala.collection.IterableFactory$ToFactory").getDeclaredField("factory"); - fieldOffset = UnsafeOps.objectFieldOffset(field); + Field field = IterableToFactoryClass.getDeclaredField("factory"); + FACTORY_ACCESSOR = FieldAccessor.createAccessor(field); + CONSTRUCTOR = IterableToFactoryClass.getDeclaredConstructor(field.getType()); + CONSTRUCTOR.setAccessible(true); } catch (final Exception e) { throw new RuntimeException(e); } @@ -55,25 +57,29 @@ public IterableToFactorySerializer(Config config) { @Override public void write(WriteContext writeContext, Object value) { - writeContext.writeRef(UnsafeOps.getObject(value, fieldOffset)); + writeContext.writeRef(FACTORY_ACCESSOR.getObject(value)); } @Override public Object read(ReadContext readContext) { - Object o = UnsafeOps.newInstance(type); - UnsafeOps.putObject(o, fieldOffset, readContext.readRef()); - return o; + try { + return CONSTRUCTOR.newInstance(readContext.readRef()); + } catch (Exception e) { + throw new ForyException("Failed to create Scala IterableFactory.ToFactory", e); + } } } public static class MapToFactorySerializer extends Serializer implements Shareable { - private static final long fieldOffset; + private static final FieldAccessor FACTORY_ACCESSOR; + private static final Constructor CONSTRUCTOR; static { try { - // for graalvm field offset auto rewrite - Field field = Class.forName("scala.collection.MapFactory$ToFactory").getDeclaredField("factory"); - fieldOffset = UnsafeOps.objectFieldOffset(field); + Field field = MapToFactoryClass.getDeclaredField("factory"); + FACTORY_ACCESSOR = FieldAccessor.createAccessor(field); + CONSTRUCTOR = MapToFactoryClass.getDeclaredConstructor(field.getType()); + CONSTRUCTOR.setAccessible(true); } catch (final Exception e) { throw new RuntimeException(e); } @@ -85,14 +91,16 @@ public MapToFactorySerializer(Config config) { @Override public void write(WriteContext writeContext, Object value) { - writeContext.writeRef(UnsafeOps.getObject(value, fieldOffset)); + writeContext.writeRef(FACTORY_ACCESSOR.getObject(value)); } @Override public Object read(ReadContext readContext) { - Object o = UnsafeOps.newInstance(type); - UnsafeOps.putObject(o, fieldOffset, readContext.readRef()); - return o; + try { + return CONSTRUCTOR.newInstance(readContext.readRef()); + } catch (Exception e) { + throw new ForyException("Failed to create Scala MapFactory.ToFactory", e); + } } } } From f1f7b2e0da703551774db3a926280d441351094b Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Mon, 25 May 2026 00:21:59 +0800 Subject: [PATCH 32/69] fix(java): restore android and graal ci --- .../serializer/ObjectStreamSerializer.java | 3 + .../fory/serializer/PlatformStringUtils.java | 61 +++++++++++++++++++ .../apache/fory/serializer/Serializers.java | 4 +- .../fory-core/native-image.properties | 1 + .../apache/fory/memory/MemoryBufferTest.java | 19 +++++- .../serializer/AndroidDynamicFeatureTest.java | 7 ++- .../serializer/AndroidJvmRoundTripTest.java | 3 + .../fory/serializer/ObjectSerializerTest.java | 7 ++- .../AndroidCollectionFeatureTest.java | 4 +- 9 files changed, 97 insertions(+), 12 deletions(-) diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectStreamSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectStreamSerializer.java index e03dd8cdf2..75e287a940 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectStreamSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectStreamSerializer.java @@ -245,6 +245,9 @@ private static Serializer fallbackSerializer(TypeResolver typeResolver, Class /** Creates an ObjectCreator for Java ObjectStream-compatible reconstruction. */ private static ObjectCreator createObjectStreamCreator(Class type) { + if (AndroidSupport.IS_ANDROID) { + return ObjectCreators.getObjectCreator(type); + } if (JdkVersion.MAJOR_VERSION >= 25) { if (hasJdk25Fallback(type)) { return new FallbackOnlyObjectCreator<>(type); diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/PlatformStringUtils.java b/java/fory-core/src/main/java/org/apache/fory/serializer/PlatformStringUtils.java index d9770e1892..5b6151b4b4 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/PlatformStringUtils.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/PlatformStringUtils.java @@ -21,6 +21,7 @@ import java.nio.charset.StandardCharsets; import org.apache.fory.memory.MemoryBuffer; +import org.apache.fory.memory.NativeByteOrder; import org.apache.fory.platform.AndroidSupport; import org.apache.fory.platform.GraalvmSupport; import org.apache.fory.platform.internal._JDKAccess; @@ -104,19 +105,75 @@ private static String newBytesStringSlow(byte coder, byte[] data) { } static long getCharsLong(char[] chars, int charIndex) { + if (AndroidSupport.IS_ANDROID) { + long c0 = chars[charIndex]; + long c1 = chars[charIndex + 1]; + long c2 = chars[charIndex + 2]; + long c3 = chars[charIndex + 3]; + if (NativeByteOrder.IS_LITTLE_ENDIAN) { + return c0 | (c1 << 16) | (c2 << 32) | (c3 << 48); + } else { + return (c0 << 48) | (c1 << 32) | (c2 << 16) | c3; + } + } return UNSAFE.getLong(chars, CHAR_ARRAY_OFFSET + ((long) charIndex << 1)); } static long getBytesLong(byte[] bytes, int byteIndex) { + if (AndroidSupport.IS_ANDROID) { + if (NativeByteOrder.IS_LITTLE_ENDIAN) { + return ((long) bytes[byteIndex] & 0xff) + | (((long) bytes[byteIndex + 1] & 0xff) << 8) + | (((long) bytes[byteIndex + 2] & 0xff) << 16) + | (((long) bytes[byteIndex + 3] & 0xff) << 24) + | (((long) bytes[byteIndex + 4] & 0xff) << 32) + | (((long) bytes[byteIndex + 5] & 0xff) << 40) + | (((long) bytes[byteIndex + 6] & 0xff) << 48) + | (((long) bytes[byteIndex + 7] & 0xff) << 56); + } else { + return (((long) bytes[byteIndex] & 0xff) << 56) + | (((long) bytes[byteIndex + 1] & 0xff) << 48) + | (((long) bytes[byteIndex + 2] & 0xff) << 40) + | (((long) bytes[byteIndex + 3] & 0xff) << 32) + | (((long) bytes[byteIndex + 4] & 0xff) << 24) + | (((long) bytes[byteIndex + 5] & 0xff) << 16) + | (((long) bytes[byteIndex + 6] & 0xff) << 8) + | ((long) bytes[byteIndex + 7] & 0xff); + } + } return UNSAFE.getLong(bytes, BYTE_ARRAY_OFFSET + byteIndex); } static char getBytesChar(byte[] bytes, int byteIndex) { + if (AndroidSupport.IS_ANDROID) { + if (NativeByteOrder.IS_LITTLE_ENDIAN) { + return (char) ((bytes[byteIndex] & 0xff) | ((bytes[byteIndex + 1] & 0xff) << 8)); + } else { + return (char) (((bytes[byteIndex] & 0xff) << 8) | (bytes[byteIndex + 1] & 0xff)); + } + } return UNSAFE.getChar(bytes, BYTE_ARRAY_OFFSET + byteIndex); } static void copyCharsToBytes( char[] chars, int charOffset, byte[] target, int byteOffset, int numBytes) { + if (AndroidSupport.IS_ANDROID) { + int charIndex = charOffset; + if (NativeByteOrder.IS_LITTLE_ENDIAN) { + for (int i = byteOffset, end = byteOffset + numBytes; i < end; i += 2) { + char c = chars[charIndex++]; + target[i] = (byte) c; + target[i + 1] = (byte) (c >>> 8); + } + } else { + for (int i = byteOffset, end = byteOffset + numBytes; i < end; i += 2) { + char c = chars[charIndex++]; + target[i] = (byte) (c >>> 8); + target[i + 1] = (byte) c; + } + } + return; + } UNSAFE.copyMemory( chars, CHAR_ARRAY_OFFSET + ((long) charOffset << 1), @@ -126,6 +183,10 @@ static void copyCharsToBytes( } static void putBytes(MemoryBuffer buffer, int writerIndex, byte[] bytes, int numBytes) { + if (AndroidSupport.IS_ANDROID) { + buffer.put(writerIndex, bytes, 0, numBytes); + return; + } long address = buffer._unsafeWriterAddress() + writerIndex - buffer.writerIndex(); UNSAFE.copyMemory(bytes, BYTE_ARRAY_OFFSET, null, address, numBytes); } diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/Serializers.java b/java/fory-core/src/main/java/org/apache/fory/serializer/Serializers.java index d8647f19d8..2e6b762948 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/Serializers.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/Serializers.java @@ -499,7 +499,7 @@ public static T read(ReadContext readContext, Serializer serializer) { private static final Function GET_VALUE; static { - if (AndroidSupport.IS_ANDROID) { + if (!MemoryUtils.JDK_LANG_FIELD_ACCESS) { GET_VALUE = null; GET_CODER = null; } else { @@ -532,7 +532,7 @@ public void write(WriteContext writeContext, T value) { stringSerializer.writeString(buffer, value.toString()); return; } - if (AndroidSupport.IS_ANDROID) { + if (!MemoryUtils.JDK_LANG_FIELD_ACCESS) { stringSerializer.writeString(buffer, value.toString()); return; } diff --git a/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties b/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties index 56c9c45adc..99024c974e 100644 --- a/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties +++ b/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties @@ -26,6 +26,7 @@ Args=--initialize-at-build-time=org.apache.fory.annotation.ForyField$Dynamic,\ org.apache.fory.collection.ReferenceConcurrentMap$SoftValueReference,\ org.apache.fory.collection.ReferenceConcurrentMap$WeakKeyReference,\ org.apache.fory.memory.ByteBufferUtil,\ + org.apache.fory.memory.LittleEndian,\ org.apache.fory.memory.MemoryBuffer,\ org.apache.fory.memory.NativeByteOrder,\ org.apache.fory.platform.AndroidSupport,\ diff --git a/java/fory-core/src/test/java/org/apache/fory/memory/MemoryBufferTest.java b/java/fory-core/src/test/java/org/apache/fory/memory/MemoryBufferTest.java index 4a30f92a55..1ed4fbbdaa 100644 --- a/java/fory-core/src/test/java/org/apache/fory/memory/MemoryBufferTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/memory/MemoryBufferTest.java @@ -307,9 +307,7 @@ public static void main(String[] args) { throw new AssertionError("Unexpected java.nio open: " + inputArg); } } - if (ByteBuffer.class - .getModule() - .isOpen("java.nio", DirectByteBufferNoNioOpenProbe.class.getModule())) { + if (isNioOpenToProbe()) { throw new AssertionError("java.base/java.nio must not be open to this test probe"); } } @@ -344,6 +342,21 @@ private static void checkEqual(long actual, long expected) { throw new AssertionError("Expected " + expected + " but got " + actual); } } + + private static boolean isNioOpenToProbe() { + try { + Class moduleType = Class.forName("java.lang.Module"); + java.lang.reflect.Method getModule = Class.class.getMethod("getModule"); + Object byteBufferModule = getModule.invoke(ByteBuffer.class); + Object probeModule = getModule.invoke(DirectByteBufferNoNioOpenProbe.class); + java.lang.reflect.Method isOpen = moduleType.getMethod("isOpen", String.class, moduleType); + return (Boolean) isOpen.invoke(byteBufferModule, "java.nio", probeModule); + } catch (ClassNotFoundException | NoSuchMethodException e) { + return false; + } catch (ReflectiveOperationException e) { + throw new AssertionError("Failed to inspect java.nio module opens", e); + } + } } @Test diff --git a/java/fory-core/src/test/java/org/apache/fory/serializer/AndroidDynamicFeatureTest.java b/java/fory-core/src/test/java/org/apache/fory/serializer/AndroidDynamicFeatureTest.java index 990f09a730..828289c39e 100644 --- a/java/fory-core/src/test/java/org/apache/fory/serializer/AndroidDynamicFeatureTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/serializer/AndroidDynamicFeatureTest.java @@ -54,10 +54,11 @@ public class AndroidDynamicFeatureTest { @Test public void testAndroidDynamicFeaturePaths() throws Exception { - Process process = + ProcessBuilder processBuilder = new ProcessBuilder(TestUtils.javaCommand(AndroidDynamicFeatureProbe.class)) - .redirectErrorStream(true) - .start(); + .redirectErrorStream(true); + processBuilder.environment().put("FORY_ANDROID_ENABLED", "1"); + Process process = processBuilder.start(); String output = readFully(process.getInputStream()); Assert.assertEquals(process.waitFor(), 0, output); } diff --git a/java/fory-core/src/test/java/org/apache/fory/serializer/AndroidJvmRoundTripTest.java b/java/fory-core/src/test/java/org/apache/fory/serializer/AndroidJvmRoundTripTest.java index 5b93035622..a003b55c00 100644 --- a/java/fory-core/src/test/java/org/apache/fory/serializer/AndroidJvmRoundTripTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/serializer/AndroidJvmRoundTripTest.java @@ -160,6 +160,9 @@ private static void addAddOpens(ArrayList command) { } addAddOpens(command, "java.base/java.io=ALL-UNNAMED"); addAddOpens(command, "java.base/java.lang=ALL-UNNAMED"); + addAddOpens(command, "java.base/java.lang.invoke=ALL-UNNAMED"); + addAddOpens(command, "java.base/java.lang.reflect=ALL-UNNAMED"); + addAddOpens(command, "java.base/jdk.internal.reflect=ALL-UNNAMED"); addAddOpens(command, "java.base/java.util=ALL-UNNAMED"); } diff --git a/java/fory-core/src/test/java/org/apache/fory/serializer/ObjectSerializerTest.java b/java/fory-core/src/test/java/org/apache/fory/serializer/ObjectSerializerTest.java index 178a1cac9e..f0b2235db3 100644 --- a/java/fory-core/src/test/java/org/apache/fory/serializer/ObjectSerializerTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/serializer/ObjectSerializerTest.java @@ -774,14 +774,15 @@ public void testSerialization() { public void testAndroidObjectSerializerReflectionPaths() throws Exception { String javaBin = System.getProperty("java.home") + File.separator + "bin" + File.separator + "java"; - Process process = + ProcessBuilder processBuilder = new ProcessBuilder( javaBin, "-cp", System.getProperty("java.class.path"), AndroidObjectSerializerProbe.class.getName()) - .redirectErrorStream(true) - .start(); + .redirectErrorStream(true); + processBuilder.environment().put("FORY_ANDROID_ENABLED", "1"); + Process process = processBuilder.start(); String output = readFully(process.getInputStream()); Assert.assertEquals(process.waitFor(), 0, output); } diff --git a/java/fory-core/src/test/java/org/apache/fory/serializer/collection/AndroidCollectionFeatureTest.java b/java/fory-core/src/test/java/org/apache/fory/serializer/collection/AndroidCollectionFeatureTest.java index 2d2e2d5736..0f6cda5edd 100644 --- a/java/fory-core/src/test/java/org/apache/fory/serializer/collection/AndroidCollectionFeatureTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/serializer/collection/AndroidCollectionFeatureTest.java @@ -85,7 +85,9 @@ public void testAndroidCollectionFeaturePaths() throws Exception { command.add(jvmSubListPayload); command.add(jvmEnumMapPayload); command.add(jvmEmptyEnumMapPayload); - Process process = new ProcessBuilder(command).redirectErrorStream(true).start(); + ProcessBuilder processBuilder = new ProcessBuilder(command).redirectErrorStream(true); + processBuilder.environment().put("FORY_ANDROID_ENABLED", "1"); + Process process = processBuilder.start(); String output = readFully(process.getInputStream()); Assert.assertEquals(process.waitFor(), 0, output); From 0ff50fcad05d40ecda3db6922dbf1f9b10ed582d Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Mon, 25 May 2026 00:50:35 +0800 Subject: [PATCH 33/69] fix(java): open arrow memory on classpath --- ci/run_ci.sh | 2 +- .../java/org/apache/fory/format/vectorized/ArrowUtils.java | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/ci/run_ci.sh b/ci/run_ci.sh index 76f9aecbd3..34434a13d8 100755 --- a/ci/run_ci.sh +++ b/ci/run_ci.sh @@ -204,7 +204,7 @@ jdk17_plus_tests() { else java_major=$(echo "$java_version" | cut -d. -f1) fi - JDK_JAVA_OPTIONS="--add-opens=java.base/java.nio=org.apache.arrow.memory.core" + JDK_JAVA_OPTIONS="--add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED" if [[ "$java_major" -ge 25 ]]; then JDK_JAVA_OPTIONS="$JDK_JAVA_OPTIONS $(jdk25_deny_options) $(jdk25_javac_options)" fi diff --git a/java/fory-format/src/main/java/org/apache/fory/format/vectorized/ArrowUtils.java b/java/fory-format/src/main/java/org/apache/fory/format/vectorized/ArrowUtils.java index 226117b7df..1daa138394 100644 --- a/java/fory-format/src/main/java/org/apache/fory/format/vectorized/ArrowUtils.java +++ b/java/fory-format/src/main/java/org/apache/fory/format/vectorized/ArrowUtils.java @@ -150,6 +150,11 @@ private static boolean isUnsafeMemoryAccessFailure(Throwable throwable) { } } } + if ("java.lang.reflect.InaccessibleObjectException".equals(cause.getClass().getName()) + && cause.getMessage() != null + && cause.getMessage().contains("java.nio")) { + return true; + } cause = cause.getCause(); } return false; From 15177584772ff7e1484616458f42c412d8f4c984 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Mon, 25 May 2026 01:15:40 +0800 Subject: [PATCH 34/69] fix(java): restore graalvm array offset recompute --- .../org/apache/fory/memory/LittleEndian.java | 12 ++++- .../org/apache/fory/memory/MemoryBuffer.java | 49 +++++++++++++------ .../fory/serializer/PlatformStringUtils.java | 18 +++++-- 3 files changed, 57 insertions(+), 22 deletions(-) diff --git a/java/fory-core/src/main/java/org/apache/fory/memory/LittleEndian.java b/java/fory-core/src/main/java/org/apache/fory/memory/LittleEndian.java index 1b58ed70d6..d3facaee67 100644 --- a/java/fory-core/src/main/java/org/apache/fory/memory/LittleEndian.java +++ b/java/fory-core/src/main/java/org/apache/fory/memory/LittleEndian.java @@ -25,8 +25,16 @@ public class LittleEndian { private static final Unsafe UNSAFE = AndroidSupport.IS_ANDROID ? null : _JDKAccess.UNSAFE; - private static final int BYTE_ARRAY_OFFSET = - AndroidSupport.IS_ANDROID ? 0 : UNSAFE.arrayBaseOffset(byte[].class); + private static final int BYTE_ARRAY_OFFSET; + + // Keep arrayBaseOffset as a direct static-field store for GraalVM native-image recomputation. + static { + if (AndroidSupport.IS_ANDROID) { + BYTE_ARRAY_OFFSET = 0; + } else { + BYTE_ARRAY_OFFSET = UNSAFE.arrayBaseOffset(byte[].class); + } + } public static int putVarUint36Small(byte[] arr, int index, long v) { if (v >>> 7 == 0) { diff --git a/java/fory-core/src/main/java/org/apache/fory/memory/MemoryBuffer.java b/java/fory-core/src/main/java/org/apache/fory/memory/MemoryBuffer.java index 6f5f8b5a8d..5b0e73617d 100644 --- a/java/fory-core/src/main/java/org/apache/fory/memory/MemoryBuffer.java +++ b/java/fory-core/src/main/java/org/apache/fory/memory/MemoryBuffer.java @@ -69,22 +69,39 @@ public final class MemoryBuffer { private static final Unsafe UNSAFE = AndroidSupport.IS_ANDROID ? null : _JDKAccess.UNSAFE; private static final boolean LITTLE_ENDIAN = NativeByteOrder.IS_LITTLE_ENDIAN; private static final boolean UNALIGNED = !AndroidSupport.IS_ANDROID && unaligned(); - private static final int BOOLEAN_ARRAY_OFFSET = - AndroidSupport.IS_ANDROID ? 0 : UNSAFE.arrayBaseOffset(boolean[].class); - private static final int BYTE_ARRAY_OFFSET = - AndroidSupport.IS_ANDROID ? 0 : UNSAFE.arrayBaseOffset(byte[].class); - private static final int CHAR_ARRAY_OFFSET = - AndroidSupport.IS_ANDROID ? 0 : UNSAFE.arrayBaseOffset(char[].class); - private static final int SHORT_ARRAY_OFFSET = - AndroidSupport.IS_ANDROID ? 0 : UNSAFE.arrayBaseOffset(short[].class); - private static final int INT_ARRAY_OFFSET = - AndroidSupport.IS_ANDROID ? 0 : UNSAFE.arrayBaseOffset(int[].class); - private static final int LONG_ARRAY_OFFSET = - AndroidSupport.IS_ANDROID ? 0 : UNSAFE.arrayBaseOffset(long[].class); - private static final int FLOAT_ARRAY_OFFSET = - AndroidSupport.IS_ANDROID ? 0 : UNSAFE.arrayBaseOffset(float[].class); - private static final int DOUBLE_ARRAY_OFFSET = - AndroidSupport.IS_ANDROID ? 0 : UNSAFE.arrayBaseOffset(double[].class); + private static final int BOOLEAN_ARRAY_OFFSET; + private static final int BYTE_ARRAY_OFFSET; + private static final int CHAR_ARRAY_OFFSET; + private static final int SHORT_ARRAY_OFFSET; + private static final int INT_ARRAY_OFFSET; + private static final int LONG_ARRAY_OFFSET; + private static final int FLOAT_ARRAY_OFFSET; + private static final int DOUBLE_ARRAY_OFFSET; + + // GraalVM native-image recognizes arrayBaseOffset only when the call stores directly into the + // target static field. Keep these assignments in this shape so native images recompute heap array + // offsets for the image runtime instead of embedding build-time VM offsets. + static { + if (AndroidSupport.IS_ANDROID) { + BOOLEAN_ARRAY_OFFSET = 0; + BYTE_ARRAY_OFFSET = 0; + CHAR_ARRAY_OFFSET = 0; + SHORT_ARRAY_OFFSET = 0; + INT_ARRAY_OFFSET = 0; + LONG_ARRAY_OFFSET = 0; + FLOAT_ARRAY_OFFSET = 0; + DOUBLE_ARRAY_OFFSET = 0; + } else { + BOOLEAN_ARRAY_OFFSET = UNSAFE.arrayBaseOffset(boolean[].class); + BYTE_ARRAY_OFFSET = UNSAFE.arrayBaseOffset(byte[].class); + CHAR_ARRAY_OFFSET = UNSAFE.arrayBaseOffset(char[].class); + SHORT_ARRAY_OFFSET = UNSAFE.arrayBaseOffset(short[].class); + INT_ARRAY_OFFSET = UNSAFE.arrayBaseOffset(int[].class); + LONG_ARRAY_OFFSET = UNSAFE.arrayBaseOffset(long[].class); + FLOAT_ARRAY_OFFSET = UNSAFE.arrayBaseOffset(float[].class); + DOUBLE_ARRAY_OFFSET = UNSAFE.arrayBaseOffset(double[].class); + } + } /** Limits each raw Unsafe copy to let large copies hit safepoint polls between chunks. */ private static final long UNSAFE_COPY_THRESHOLD = 1024L * 1024L; diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/PlatformStringUtils.java b/java/fory-core/src/main/java/org/apache/fory/serializer/PlatformStringUtils.java index 5b6151b4b4..e2335af192 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/PlatformStringUtils.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/PlatformStringUtils.java @@ -30,10 +30,20 @@ /** Platform-owned string internals used by {@link StringSerializer}. */ final class PlatformStringUtils { private static final Unsafe UNSAFE = AndroidSupport.IS_ANDROID ? null : _JDKAccess.UNSAFE; - private static final int BYTE_ARRAY_OFFSET = - AndroidSupport.IS_ANDROID ? 0 : UNSAFE.arrayBaseOffset(byte[].class); - private static final int CHAR_ARRAY_OFFSET = - AndroidSupport.IS_ANDROID ? 0 : UNSAFE.arrayBaseOffset(char[].class); + private static final int BYTE_ARRAY_OFFSET; + private static final int CHAR_ARRAY_OFFSET; + + // GraalVM native-image needs arrayBaseOffset calls to store directly into their static fields so + // it can recompute the offsets for the image runtime. + static { + if (AndroidSupport.IS_ANDROID) { + BYTE_ARRAY_OFFSET = 0; + CHAR_ARRAY_OFFSET = 0; + } else { + BYTE_ARRAY_OFFSET = UNSAFE.arrayBaseOffset(byte[].class); + CHAR_ARRAY_OFFSET = UNSAFE.arrayBaseOffset(char[].class); + } + } static final boolean JDK_STRING_FIELD_ACCESS = !AndroidSupport.IS_ANDROID From 63ad7e67fd6377f2df4a3fefbd02cc3d699b1375 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Mon, 1 Jun 2026 18:10:50 +0800 Subject: [PATCH 35/69] refactor(java): remove byte array stream private wrapping --- ci/run_ci.sh | 4 - docs/guide/java/troubleshooting.md | 19 +- integration_tests/graalvm_tests/pom.xml | 8 - .../jdk_compatibility_tests/pom.xml | 4 - integration_tests/jpms_tests/pom.xml | 4 - .../src/main/java/org/apache/fory/Fory.java | 38 +-- .../org/apache/fory/memory/MemoryUtils.java | 46 ---- .../fory/platform/internal/_JDKAccess.java | 149 ++++------- .../serializer/ObjectStreamSerializer.java | 26 +- .../serializer/ReplaceResolveSerializer.java | 19 +- .../fory/platform/internal/_JDKAccess.java | 253 +++++------------- .../fory-core/native-image.properties | 1 - .../test/java/org/apache/fory/TestUtils.java | 4 - .../apache/fory/memory/MemoryBufferTest.java | 24 -- .../serializer/AndroidDynamicFeatureTest.java | 27 +- .../serializer/AndroidJvmRoundTripTest.java | 1 - java/fory-format/pom.xml | 10 - scala/build.sbt | 4 - 18 files changed, 181 insertions(+), 460 deletions(-) diff --git a/ci/run_ci.sh b/ci/run_ci.sh index 5c5f713eaf..a742161d8e 100755 --- a/ci/run_ci.sh +++ b/ci/run_ci.sh @@ -110,10 +110,6 @@ jdk25_deny_options() { printf " %s" "--add-opens=java.base/jdk.internal.reflect=${fory_open_targets}" printf " %s" "--add-opens=java.base/java.util=${fory_open_targets}" printf " %s" "--add-opens=java.base/java.util.concurrent=${fory_open_targets}" - printf " %s" "--add-opens=java.base/java.util.concurrent.atomic=${fory_open_targets}" - printf " %s" "--add-opens=java.base/java.io=${fory_open_targets}" - printf " %s" "--add-opens=java.base/java.net=${fory_open_targets}" - printf " %s" "--add-opens=java.base/java.math=${fory_open_targets}" } jdk25_javac_options() { diff --git a/docs/guide/java/troubleshooting.md b/docs/guide/java/troubleshooting.md index 9c35b3eb70..517af28ce5 100644 --- a/docs/guide/java/troubleshooting.md +++ b/docs/guide/java/troubleshooting.md @@ -169,17 +169,14 @@ owning JDK module/package to `org.apache.fory.core` and `org.apache.fory.format` `ALL-UNNAMED` too when any Fory artifact is on the classpath. Add only the opens needed by the paths used in your process: -| Path | Required opens | -| -------------------------------------------------------------------------------- | ------------------------------------------------------------------------- | -| String fast paths and throwable fields | `java.base/java.lang` | -| Serialized lambdas | `java.base/java.lang.invoke` | -| Reflection-based object construction | `java.base/java.lang.reflect`, `java.base/jdk.internal.reflect` | -| Collection wrappers, sublists, `EnumMap`, and `StringTokenizer` | `java.base/java.util` | -| Blocking queue capacity serializers | `java.base/java.util.concurrent`, `java.base/java.util.concurrent.atomic` | -| `ByteArrayInputStream`, `ByteArrayOutputStream`, and Java object-stream metadata | `java.base/java.io` | -| URL and networking serializers | `java.base/java.net` | -| Proxy serializers | `java.base/java.lang.reflect` | -| Big number internals | `java.base/java.math` | +| Path | Required opens | +| --------------------------------------------------------------- | --------------------------------------------------------------- | +| String fast paths and throwable fields | `java.base/java.lang` | +| Serialized lambdas | `java.base/java.lang.invoke` | +| Reflection-based object construction | `java.base/java.lang.reflect`, `java.base/jdk.internal.reflect` | +| Collection wrappers, sublists, `EnumMap`, and `StringTokenizer` | `java.base/java.util` | +| Blocking queue capacity serializers | `java.base/java.util.concurrent` | +| Proxy serializers | `java.base/java.lang.reflect` | Normal classes with final instance fields require final-field mutation to be enabled for the module that contains Fory's mutating code when Unsafe allocation is denied. Use the Fory module name on the diff --git a/integration_tests/graalvm_tests/pom.xml b/integration_tests/graalvm_tests/pom.xml index 64cfc7ceff..c2be9cd979 100644 --- a/integration_tests/graalvm_tests/pom.xml +++ b/integration_tests/graalvm_tests/pom.xml @@ -182,10 +182,6 @@ -J--add-opens=java.base/jdk.internal.reflect=ALL-UNNAMED -J--add-opens=java.base/java.util=ALL-UNNAMED -J--add-opens=java.base/java.util.concurrent=ALL-UNNAMED - -J--add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED - -J--add-opens=java.base/java.io=ALL-UNNAMED - -J--add-opens=java.base/java.net=ALL-UNNAMED - -J--add-opens=java.base/java.math=ALL-UNNAMED @@ -209,10 +205,6 @@ --add-opens=java.base/jdk.internal.reflect=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED - --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED - --add-opens=java.base/java.io=ALL-UNNAMED - --add-opens=java.base/java.net=ALL-UNNAMED - --add-opens=java.base/java.math=ALL-UNNAMED -classpath ${mainClass} diff --git a/integration_tests/jdk_compatibility_tests/pom.xml b/integration_tests/jdk_compatibility_tests/pom.xml index 81c3c97214..6dd1850b8d 100644 --- a/integration_tests/jdk_compatibility_tests/pom.xml +++ b/integration_tests/jdk_compatibility_tests/pom.xml @@ -101,10 +101,6 @@ --add-opens=java.base/jdk.internal.reflect=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED - --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED - --add-opens=java.base/java.io=ALL-UNNAMED - --add-opens=java.base/java.net=ALL-UNNAMED - --add-opens=java.base/java.math=ALL-UNNAMED diff --git a/integration_tests/jpms_tests/pom.xml b/integration_tests/jpms_tests/pom.xml index a2e83e755c..5f229a57b8 100644 --- a/integration_tests/jpms_tests/pom.xml +++ b/integration_tests/jpms_tests/pom.xml @@ -96,10 +96,6 @@ --add-opens=java.base/jdk.internal.reflect=org.apache.fory.core,org.apache.fory.format --add-opens=java.base/java.util=org.apache.fory.core,org.apache.fory.format --add-opens=java.base/java.util.concurrent=org.apache.fory.core,org.apache.fory.format - --add-opens=java.base/java.util.concurrent.atomic=org.apache.fory.core,org.apache.fory.format - --add-opens=java.base/java.io=org.apache.fory.core,org.apache.fory.format - --add-opens=java.base/java.net=org.apache.fory.core,org.apache.fory.format - --add-opens=java.base/java.math=org.apache.fory.core,org.apache.fory.format diff --git a/java/fory-core/src/main/java/org/apache/fory/Fory.java b/java/fory-core/src/main/java/org/apache/fory/Fory.java index 3ad373043b..d47cbc4e5c 100644 --- a/java/fory-core/src/main/java/org/apache/fory/Fory.java +++ b/java/fory-core/src/main/java/org/apache/fory/Fory.java @@ -19,7 +19,6 @@ package org.apache.fory; -import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.OutputStream; import java.nio.ByteBuffer; @@ -563,31 +562,20 @@ public T copy(T obj) { private void serializeToStream(OutputStream outputStream, Consumer function) { MemoryBuffer buf = getBuffer(); - if (MemoryUtils.JDK_BYTE_ARRAY_STREAM_FIELD_ACCESS - && outputStream.getClass() == ByteArrayOutputStream.class) { - byte[] oldBytes = buf.getHeapMemory(); // Note: This should not be null. - assert oldBytes != null; - MemoryUtils.wrap((ByteArrayOutputStream) outputStream, buf); - function.accept(buf); - MemoryUtils.wrap(buf, (ByteArrayOutputStream) outputStream); - buf.pointTo(oldBytes, 0, oldBytes.length); - resetBuffer(); - } else { - buf.writerIndex(0); - function.accept(buf); - try { - byte[] bytes = buf.getHeapMemory(); - if (bytes != null) { - outputStream.write(bytes, 0, buf.writerIndex()); - } else { - outputStream.write(buf.getBytes(0, buf.writerIndex())); - } - outputStream.flush(); - } catch (IOException e) { - throw new SerializationException(e); - } finally { - resetBuffer(); + buf.writerIndex(0); + function.accept(buf); + try { + byte[] bytes = buf.getHeapMemory(); + if (bytes != null) { + outputStream.write(bytes, 0, buf.writerIndex()); + } else { + outputStream.write(buf.getBytes(0, buf.writerIndex())); } + outputStream.flush(); + } catch (IOException e) { + throw new SerializationException(e); + } finally { + resetBuffer(); } } diff --git a/java/fory-core/src/main/java/org/apache/fory/memory/MemoryUtils.java b/java/fory-core/src/main/java/org/apache/fory/memory/MemoryUtils.java index 41db2cfe46..99eda09059 100644 --- a/java/fory-core/src/main/java/org/apache/fory/memory/MemoryUtils.java +++ b/java/fory-core/src/main/java/org/apache/fory/memory/MemoryUtils.java @@ -19,8 +19,6 @@ package org.apache.fory.memory; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; import java.nio.ByteBuffer; import org.apache.fory.platform.AndroidSupport; import org.apache.fory.platform.GraalvmSupport; @@ -39,14 +37,6 @@ public class MemoryUtils { !AndroidSupport.IS_ANDROID && !GraalvmSupport.IN_GRAALVM_NATIVE_IMAGE && _JDKAccess.JDK_LANG_FIELD_ACCESS; - public static final boolean JDK_BYTE_ARRAY_STREAM_FIELD_ACCESS = - !AndroidSupport.IS_ANDROID - && !GraalvmSupport.IN_GRAALVM_NATIVE_IMAGE - && _JDKAccess.JDK_BYTE_ARRAY_STREAM_FIELD_ACCESS; - public static final boolean JDK_OBJECT_STREAM_FIELD_ACCESS = - !AndroidSupport.IS_ANDROID - && !GraalvmSupport.IN_GRAALVM_NATIVE_IMAGE - && _JDKAccess.JDK_OBJECT_STREAM_FIELD_ACCESS; public static final boolean JDK_COLLECTION_FIELD_ACCESS = !AndroidSupport.IS_ANDROID && !GraalvmSupport.IN_GRAALVM_NATIVE_IMAGE @@ -102,42 +92,6 @@ public static MemoryBuffer wrap(ByteBuffer buffer) { } } - /** - * Wrap a {@link ByteArrayOutputStream} into a {@link MemoryBuffer}. The writerIndex of buffer - * will be the count of stream. - */ - public static void wrap(ByteArrayOutputStream stream, MemoryBuffer buffer) { - checkByteArrayStreamWrap("ByteArrayOutputStream"); - _JDKAccess.wrap(stream, buffer); - } - - /** - * Wrap a @link MemoryBuffer} into a {@link ByteArrayOutputStream}. The count of stream will be - * the writerIndex of buffer. - */ - public static void wrap(MemoryBuffer buffer, ByteArrayOutputStream stream) { - checkByteArrayStreamWrap("ByteArrayOutputStream"); - _JDKAccess.wrap(buffer, stream); - } - - /** - * Wrap a {@link ByteArrayInputStream} into a {@link MemoryBuffer}. The readerIndex of buffer will - * be the pos of stream. - */ - public static void wrap(ByteArrayInputStream stream, MemoryBuffer buffer) { - checkByteArrayStreamWrap("ByteArrayInputStream"); - _JDKAccess.wrap(stream, buffer); - } - - private static void checkByteArrayStreamWrap(String streamType) { - if (!JDK_BYTE_ARRAY_STREAM_FIELD_ACCESS) { - throw new UnsupportedOperationException( - streamType - + " direct wrapping requires JDK internal field access. On JDK25+, open " - + "java.base/java.io to org.apache.fory.core,org.apache.fory.format."); - } - } - private static MemoryBuffer copyToHeapBuffer(ByteBuffer buffer) { ByteBuffer duplicate = buffer.duplicate(); byte[] bytes = new byte[duplicate.remaining()]; diff --git a/java/fory-core/src/main/java/org/apache/fory/platform/internal/_JDKAccess.java b/java/fory-core/src/main/java/org/apache/fory/platform/internal/_JDKAccess.java index 9efba19593..1c971802c2 100644 --- a/java/fory-core/src/main/java/org/apache/fory/platform/internal/_JDKAccess.java +++ b/java/fory-core/src/main/java/org/apache/fory/platform/internal/_JDKAccess.java @@ -19,9 +19,6 @@ package org.apache.fory.platform.internal; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.ObjectStreamClass; import java.lang.invoke.CallSite; import java.lang.invoke.LambdaMetafactory; import java.lang.invoke.MethodHandle; @@ -44,7 +41,6 @@ import java.util.function.ToLongFunction; import org.apache.fory.collection.ClassValueCache; import org.apache.fory.collection.Tuple2; -import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.platform.AndroidSupport; import org.apache.fory.platform.GraalvmSupport; import org.apache.fory.platform.JdkVersion; @@ -69,8 +65,6 @@ public class _JDKAccess { public static final boolean JDK_INTERNAL_FIELD_ACCESS; public static final boolean JDK_LANG_FIELD_ACCESS; public static final boolean JDK_STRING_FIELD_ACCESS; - public static final boolean JDK_BYTE_ARRAY_STREAM_FIELD_ACCESS; - public static final boolean JDK_OBJECT_STREAM_FIELD_ACCESS; public static final boolean JDK_COLLECTION_FIELD_ACCESS; public static final boolean JDK_CONCURRENT_FIELD_ACCESS; public static final boolean JDK_PROXY_FIELD_ACCESS; @@ -93,8 +87,6 @@ public class _JDKAccess { JDK_INTERNAL_FIELD_ACCESS = false; JDK_LANG_FIELD_ACCESS = false; JDK_STRING_FIELD_ACCESS = false; - JDK_BYTE_ARRAY_STREAM_FIELD_ACCESS = false; - JDK_OBJECT_STREAM_FIELD_ACCESS = false; JDK_COLLECTION_FIELD_ACCESS = false; JDK_CONCURRENT_FIELD_ACCESS = false; JDK_PROXY_FIELD_ACCESS = false; @@ -102,8 +94,6 @@ public class _JDKAccess { JDK_INTERNAL_FIELD_ACCESS = true; JDK_LANG_FIELD_ACCESS = true; JDK_STRING_FIELD_ACCESS = true; - JDK_BYTE_ARRAY_STREAM_FIELD_ACCESS = true; - JDK_OBJECT_STREAM_FIELD_ACCESS = true; JDK_COLLECTION_FIELD_ACCESS = true; JDK_CONCURRENT_FIELD_ACCESS = true; JDK_PROXY_FIELD_ACCESS = true; @@ -364,105 +354,84 @@ private static MethodHandle getJavaStringZeroCopyCtrHandle() { } } - // Lazy load offsets and keep the access shape in one class so the JDK25 multi-release - // replacement can change these methods without touching MemoryUtils callers. - private static class ByteArrayStreamFields { - private static final long BAS_BUF_BUF; - private static final long BAS_BUF_COUNT; - private static final long BIS_BUF_BUF; - private static final long BIS_BUF_POS; - private static final long BIS_BUF_COUNT; + private static class SerializationMethods { + private static final Object REFLECTION_FACTORY; + private static final Method WRITE_OBJECT; + private static final Method READ_OBJECT; + private static final Method READ_OBJECT_NO_DATA; + private static final Method WRITE_REPLACE; + private static final Method READ_RESOLVE; static { + Object reflectionFactory = null; + Method writeObject = null; + Method readObject = null; + Method readObjectNoData = null; + Method writeReplace = null; + Method readResolve = null; try { - BAS_BUF_BUF = UNSAFE.objectFieldOffset(ByteArrayOutputStream.class.getDeclaredField("buf")); - BAS_BUF_COUNT = - UNSAFE.objectFieldOffset(ByteArrayOutputStream.class.getDeclaredField("count")); - BIS_BUF_BUF = UNSAFE.objectFieldOffset(ByteArrayInputStream.class.getDeclaredField("buf")); - BIS_BUF_POS = UNSAFE.objectFieldOffset(ByteArrayInputStream.class.getDeclaredField("pos")); - BIS_BUF_COUNT = - UNSAFE.objectFieldOffset(ByteArrayInputStream.class.getDeclaredField("count")); - } catch (NoSuchFieldException e) { - throw new RuntimeException(e); + Class factoryClass = Class.forName("sun.reflect.ReflectionFactory"); + Method getReflectionFactory = factoryClass.getDeclaredMethod("getReflectionFactory"); + reflectionFactory = getReflectionFactory.invoke(null); + writeObject = factoryClass.getDeclaredMethod("writeObjectForSerialization", Class.class); + readObject = factoryClass.getDeclaredMethod("readObjectForSerialization", Class.class); + readObjectNoData = + factoryClass.getDeclaredMethod("readObjectNoDataForSerialization", Class.class); + writeReplace = factoryClass.getDeclaredMethod("writeReplaceForSerialization", Class.class); + readResolve = factoryClass.getDeclaredMethod("readResolveForSerialization", Class.class); + } catch (Throwable e) { + ExceptionUtils.ignore(e); } + REFLECTION_FACTORY = reflectionFactory; + WRITE_OBJECT = writeObject; + READ_OBJECT = readObject; + READ_OBJECT_NO_DATA = readObjectNoData; + WRITE_REPLACE = writeReplace; + READ_RESOLVE = readResolve; } } - public static void wrap(ByteArrayOutputStream stream, MemoryBuffer buffer) { - Preconditions.checkNotNull(stream); - byte[] buf = (byte[]) UNSAFE.getObject(stream, ByteArrayStreamFields.BAS_BUF_BUF); - int count = UNSAFE.getInt(stream, ByteArrayStreamFields.BAS_BUF_COUNT); - buffer.pointTo(buf, 0, buf.length); - buffer.writerIndex(count); - } - - public static void wrap(MemoryBuffer buffer, ByteArrayOutputStream stream) { - Preconditions.checkNotNull(stream); - byte[] bytes = buffer.getHeapMemory(); - Preconditions.checkNotNull(bytes); - UNSAFE.putObject(stream, ByteArrayStreamFields.BAS_BUF_BUF, bytes); - UNSAFE.putInt(stream, ByteArrayStreamFields.BAS_BUF_COUNT, buffer.writerIndex()); - } - - public static void wrap(ByteArrayInputStream stream, MemoryBuffer buffer) { - Preconditions.checkNotNull(stream); - byte[] buf = (byte[]) UNSAFE.getObject(stream, ByteArrayStreamFields.BIS_BUF_BUF); - int count = UNSAFE.getInt(stream, ByteArrayStreamFields.BIS_BUF_COUNT); - int pos = UNSAFE.getInt(stream, ByteArrayStreamFields.BIS_BUF_POS); - buffer.pointTo(buf, 0, count); - buffer.readerIndex(pos); + private static Method getSerializationMethod(Class type, Method factoryMethod) { + if (!isSerializationHookLookupAvailable() || factoryMethod == null) { + return null; + } + try { + MethodHandle handle = + (MethodHandle) factoryMethod.invoke(SerializationMethods.REFLECTION_FACTORY, type); + return handle == null ? null : MethodHandles.reflectAs(Method.class, handle); + } catch (Throwable e) { + ExceptionUtils.ignore(e); + return null; + } } - private static class ObjectStreamClassFields { - private static final long WRITE_OBJECT_METHOD; - private static final long READ_OBJECT_METHOD; - private static final long READ_OBJECT_NO_DATA_METHOD; - private static final long WRITE_REPLACE_METHOD; - private static final long READ_RESOLVE_METHOD; - - static { - try { - WRITE_OBJECT_METHOD = - UNSAFE.objectFieldOffset(ObjectStreamClass.class.getDeclaredField("writeObjectMethod")); - READ_OBJECT_METHOD = - UNSAFE.objectFieldOffset(ObjectStreamClass.class.getDeclaredField("readObjectMethod")); - READ_OBJECT_NO_DATA_METHOD = - UNSAFE.objectFieldOffset( - ObjectStreamClass.class.getDeclaredField("readObjectNoDataMethod")); - WRITE_REPLACE_METHOD = - UNSAFE.objectFieldOffset( - ObjectStreamClass.class.getDeclaredField("writeReplaceMethod")); - READ_RESOLVE_METHOD = - UNSAFE.objectFieldOffset(ObjectStreamClass.class.getDeclaredField("readResolveMethod")); - } catch (NoSuchFieldException e) { - throw new RuntimeException(e); - } - } + public static Method getSerializationWriteObjectMethod(Class type) { + return getSerializationMethod(type, SerializationMethods.WRITE_OBJECT); } - public static Method getObjectStreamClassWriteObjectMethod(ObjectStreamClass objectStreamClass) { - return (Method) - UNSAFE.getObject(objectStreamClass, ObjectStreamClassFields.WRITE_OBJECT_METHOD); + public static Method getSerializationReadObjectMethod(Class type) { + return getSerializationMethod(type, SerializationMethods.READ_OBJECT); } - public static Method getObjectStreamClassReadObjectMethod(ObjectStreamClass objectStreamClass) { - return (Method) UNSAFE.getObject(objectStreamClass, ObjectStreamClassFields.READ_OBJECT_METHOD); + public static Method getSerializationReadObjectNoDataMethod(Class type) { + return getSerializationMethod(type, SerializationMethods.READ_OBJECT_NO_DATA); } - public static Method getObjectStreamClassReadObjectNoDataMethod( - ObjectStreamClass objectStreamClass) { - return (Method) - UNSAFE.getObject(objectStreamClass, ObjectStreamClassFields.READ_OBJECT_NO_DATA_METHOD); + public static Method getSerializationWriteReplaceMethod(Class type) { + return getSerializationMethod(type, SerializationMethods.WRITE_REPLACE); } - public static Method getObjectStreamClassWriteReplaceMethod(ObjectStreamClass objectStreamClass) { - return (Method) - UNSAFE.getObject(objectStreamClass, ObjectStreamClassFields.WRITE_REPLACE_METHOD); + public static Method getSerializationReadResolveMethod(Class type) { + return getSerializationMethod(type, SerializationMethods.READ_RESOLVE); } - public static Method getObjectStreamClassReadResolveMethod(ObjectStreamClass objectStreamClass) { - return (Method) - UNSAFE.getObject(objectStreamClass, ObjectStreamClassFields.READ_RESOLVE_METHOD); + public static boolean isSerializationHookLookupAvailable() { + return SerializationMethods.REFLECTION_FACTORY != null + && SerializationMethods.WRITE_OBJECT != null + && SerializationMethods.READ_OBJECT != null + && SerializationMethods.READ_OBJECT_NO_DATA != null + && SerializationMethods.WRITE_REPLACE != null + && SerializationMethods.READ_RESOLVE != null; } public static T tryMakeFunction( diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectStreamSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectStreamSerializer.java index 8a74583574..631541e5d9 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectStreamSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectStreamSerializer.java @@ -58,7 +58,6 @@ import org.apache.fory.logging.Logger; import org.apache.fory.logging.LoggerFactory; import org.apache.fory.memory.MemoryBuffer; -import org.apache.fory.memory.MemoryUtils; import org.apache.fory.meta.FieldInfo; import org.apache.fory.meta.FieldTypes; import org.apache.fory.meta.NativeTypeDefEncoder; @@ -686,21 +685,28 @@ private static class StreamTypeInfo { private final Consumer readObjectNoDataFunc; private StreamTypeInfo(Class type) { - // ObjectStreamClass.lookup has cache inside, invocation cost won't be big. - ObjectStreamClass objectStreamClass = safeObjectStreamClassLookup(type); - // In JDK17, set private jdk method accessible will fail by default, use ObjectStreamClass - // instead, since it set accessible. + // ReflectionFactory exposes Java serialization hooks without reading ObjectStreamClass + // private fields or requiring the java.io package to be opened. Method writeMethod = null; Method readMethod = null; Method noDataMethod = null; - if (AndroidSupport.IS_ANDROID || !MemoryUtils.JDK_OBJECT_STREAM_FIELD_ACCESS) { + if (AndroidSupport.IS_ANDROID || GraalvmSupport.IN_GRAALVM_NATIVE_IMAGE) { writeMethod = JavaSerializer.getWriteObjectMethod(type, false); readMethod = JavaSerializer.getReadRefMethod(type, false); noDataMethod = JavaSerializer.getReadRefNoData(type, false); - } else if (objectStreamClass != null) { - writeMethod = _JDKAccess.getObjectStreamClassWriteObjectMethod(objectStreamClass); - readMethod = _JDKAccess.getObjectStreamClassReadObjectMethod(objectStreamClass); - noDataMethod = _JDKAccess.getObjectStreamClassReadObjectNoDataMethod(objectStreamClass); + } else { + writeMethod = _JDKAccess.getSerializationWriteObjectMethod(type); + readMethod = _JDKAccess.getSerializationReadObjectMethod(type); + noDataMethod = _JDKAccess.getSerializationReadObjectNoDataMethod(type); + if (writeMethod == null) { + writeMethod = JavaSerializer.getWriteObjectMethod(type, false); + } + if (readMethod == null) { + readMethod = JavaSerializer.getReadRefMethod(type, false); + } + if (noDataMethod == null) { + noDataMethod = JavaSerializer.getReadRefNoData(type, false); + } } this.writeObjectMethod = writeMethod; this.readObjectMethod = readMethod; diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/ReplaceResolveSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/ReplaceResolveSerializer.java index 598977c545..525287521b 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/ReplaceResolveSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/ReplaceResolveSerializer.java @@ -20,7 +20,6 @@ package org.apache.fory.serializer; import java.io.Externalizable; -import java.io.ObjectStreamClass; import java.io.Serializable; import java.lang.invoke.MethodHandles; import java.lang.reflect.Method; @@ -36,8 +35,8 @@ import org.apache.fory.logging.Logger; import org.apache.fory.logging.LoggerFactory; import org.apache.fory.memory.MemoryBuffer; -import org.apache.fory.memory.MemoryUtils; import org.apache.fory.platform.AndroidSupport; +import org.apache.fory.platform.GraalvmSupport; import org.apache.fory.platform.JdkVersion; import org.apache.fory.platform.internal._JDKAccess; import org.apache.fory.reflect.ObjectCreators; @@ -75,15 +74,17 @@ protected static class ReplaceResolveInfo { private ReplaceResolveInfo(Class cls) { Method writeReplaceMethod, readResolveMethod; - // In JDK17, set private jdk method accessible will fail by default, use ObjectStreamClass - // instead, since it set accessible. - if (AndroidSupport.IS_ANDROID || !MemoryUtils.JDK_OBJECT_STREAM_FIELD_ACCESS) { + if (AndroidSupport.IS_ANDROID || GraalvmSupport.IN_GRAALVM_NATIVE_IMAGE) { writeReplaceMethod = JavaSerializer.getWriteReplaceMethod(cls); readResolveMethod = JavaSerializer.getReadResolveMethod(cls); } else if (Serializable.class.isAssignableFrom(cls)) { - ObjectStreamClass objectStreamClass = ObjectStreamClass.lookup(cls); - writeReplaceMethod = _JDKAccess.getObjectStreamClassWriteReplaceMethod(objectStreamClass); - readResolveMethod = _JDKAccess.getObjectStreamClassReadResolveMethod(objectStreamClass); + if (_JDKAccess.isSerializationHookLookupAvailable()) { + writeReplaceMethod = _JDKAccess.getSerializationWriteReplaceMethod(cls); + readResolveMethod = _JDKAccess.getSerializationReadResolveMethod(cls); + } else { + writeReplaceMethod = JavaSerializer.getWriteReplaceMethod(cls); + readResolveMethod = JavaSerializer.getReadResolveMethod(cls); + } } else { // FIXME class with `writeReplace` method defined should be Serializable, // but hessian ignores this check and many existing system are using hessian, @@ -113,7 +114,7 @@ private ReplaceResolveInfo(Class cls) { : (readResolveMethod != null ? readResolveMethod.getDeclaringClass() : null); Function writeReplaceFunc = null, readResolveFunc = null; if (declaringClass != null) { - if (AndroidSupport.IS_ANDROID || !MemoryUtils.JDK_OBJECT_STREAM_FIELD_ACCESS) { + if (AndroidSupport.IS_ANDROID || GraalvmSupport.IN_GRAALVM_NATIVE_IMAGE) { makeAccessible(writeReplaceMethod); makeAccessible(readResolveMethod); } else { diff --git a/java/fory-core/src/main/java25/org/apache/fory/platform/internal/_JDKAccess.java b/java/fory-core/src/main/java25/org/apache/fory/platform/internal/_JDKAccess.java index 56caf8e197..45ac0a3790 100644 --- a/java/fory-core/src/main/java25/org/apache/fory/platform/internal/_JDKAccess.java +++ b/java/fory-core/src/main/java25/org/apache/fory/platform/internal/_JDKAccess.java @@ -19,9 +19,6 @@ package org.apache.fory.platform.internal; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.ObjectStreamClass; import java.lang.invoke.CallSite; import java.lang.invoke.LambdaConversionException; import java.lang.invoke.LambdaMetafactory; @@ -49,7 +46,6 @@ import java.util.function.ToLongFunction; import org.apache.fory.collection.ClassValueCache; import org.apache.fory.collection.Tuple2; -import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.platform.GraalvmSupport; import org.apache.fory.platform.JdkVersion; import org.apache.fory.type.TypeUtils; @@ -70,8 +66,6 @@ public class _JDKAccess { public static final boolean JDK_INTERNAL_FIELD_ACCESS; public static final boolean JDK_LANG_FIELD_ACCESS; public static final boolean JDK_STRING_FIELD_ACCESS; - public static final boolean JDK_BYTE_ARRAY_STREAM_FIELD_ACCESS; - public static final boolean JDK_OBJECT_STREAM_FIELD_ACCESS; public static final boolean JDK_COLLECTION_FIELD_ACCESS; public static final boolean JDK_CONCURRENT_FIELD_ACCESS; public static final boolean JDK_PROXY_FIELD_ACCESS; @@ -95,16 +89,6 @@ public static Unsafe unsafe() { private static final VarHandle STRING_CODER_HANDLE; private static final VarHandle STRING_COUNT_HANDLE; private static final VarHandle STRING_OFFSET_HANDLE; - private static final VarHandle BAS_BUF_HANDLE; - private static final VarHandle BAS_COUNT_HANDLE; - private static final VarHandle BIS_BUF_HANDLE; - private static final VarHandle BIS_POS_HANDLE; - private static final VarHandle BIS_COUNT_HANDLE; - private static final VarHandle OSC_WRITE_OBJECT_METHOD_HANDLE; - private static final VarHandle OSC_READ_OBJECT_METHOD_HANDLE; - private static final VarHandle OSC_READ_OBJECT_NO_DATA_METHOD_HANDLE; - private static final VarHandle OSC_WRITE_REPLACE_METHOD_HANDLE; - private static final VarHandle OSC_READ_RESOLVE_METHOD_HANDLE; static { String jmvName = System.getProperty("java.vm.name", ""); @@ -130,41 +114,19 @@ public static Unsafe unsafe() { } StringHandles stringHandles = initStringHandles(valueField.getType(), countField, offsetField); - StreamHandles streamHandles = initStreamHandles(); - ObjectStreamHandles objectStreamHandles = initObjectStreamHandles(); JDK_LANG_FIELD_ACCESS = canOpen(String.class); JDK_STRING_FIELD_ACCESS = stringHandles != null; - JDK_BYTE_ARRAY_STREAM_FIELD_ACCESS = streamHandles != null; - JDK_OBJECT_STREAM_FIELD_ACCESS = objectStreamHandles != null; JDK_COLLECTION_FIELD_ACCESS = canOpen("java.util.Collections$SynchronizedCollection"); JDK_CONCURRENT_FIELD_ACCESS = canOpen(ArrayBlockingQueue.class) && canOpen(LinkedBlockingQueue.class); JDK_PROXY_FIELD_ACCESS = canOpen(Proxy.class); - JDK_INTERNAL_FIELD_ACCESS = - JDK_STRING_FIELD_ACCESS - && JDK_BYTE_ARRAY_STREAM_FIELD_ACCESS - && JDK_OBJECT_STREAM_FIELD_ACCESS; + JDK_INTERNAL_FIELD_ACCESS = JDK_STRING_FIELD_ACCESS; STRING_VALUE_HANDLE = stringHandles == null ? null : stringHandles.value; STRING_CODER_HANDLE = stringHandles == null ? null : stringHandles.coder; STRING_COUNT_HANDLE = stringHandles == null ? null : stringHandles.count; STRING_OFFSET_HANDLE = stringHandles == null ? null : stringHandles.offset; - BAS_BUF_HANDLE = streamHandles == null ? null : streamHandles.basBuf; - BAS_COUNT_HANDLE = streamHandles == null ? null : streamHandles.basCount; - BIS_BUF_HANDLE = streamHandles == null ? null : streamHandles.bisBuf; - BIS_POS_HANDLE = streamHandles == null ? null : streamHandles.bisPos; - BIS_COUNT_HANDLE = streamHandles == null ? null : streamHandles.bisCount; - OSC_WRITE_OBJECT_METHOD_HANDLE = - objectStreamHandles == null ? null : objectStreamHandles.writeObjectMethod; - OSC_READ_OBJECT_METHOD_HANDLE = - objectStreamHandles == null ? null : objectStreamHandles.readObjectMethod; - OSC_READ_OBJECT_NO_DATA_METHOD_HANDLE = - objectStreamHandles == null ? null : objectStreamHandles.readObjectNoDataMethod; - OSC_WRITE_REPLACE_METHOD_HANDLE = - objectStreamHandles == null ? null : objectStreamHandles.writeReplaceMethod; - OSC_READ_RESOLVE_METHOD_HANDLE = - objectStreamHandles == null ? null : objectStreamHandles.readResolveMethod; } catch (NoSuchFieldException e) { throw new RuntimeException(e); } @@ -196,38 +158,6 @@ private static StringHandles initStringHandles( } } - private static StreamHandles initStreamHandles() { - try { - Lookup basLookup = - MethodHandles.privateLookupIn(ByteArrayOutputStream.class, MethodHandles.lookup()); - Lookup bisLookup = - MethodHandles.privateLookupIn(ByteArrayInputStream.class, MethodHandles.lookup()); - return new StreamHandles( - basLookup.findVarHandle(ByteArrayOutputStream.class, "buf", byte[].class), - basLookup.findVarHandle(ByteArrayOutputStream.class, "count", int.class), - bisLookup.findVarHandle(ByteArrayInputStream.class, "buf", byte[].class), - bisLookup.findVarHandle(ByteArrayInputStream.class, "pos", int.class), - bisLookup.findVarHandle(ByteArrayInputStream.class, "count", int.class)); - } catch (Throwable ignored) { - return null; - } - } - - private static ObjectStreamHandles initObjectStreamHandles() { - try { - Lookup oscLookup = - MethodHandles.privateLookupIn(ObjectStreamClass.class, MethodHandles.lookup()); - return new ObjectStreamHandles( - oscLookup.findVarHandle(ObjectStreamClass.class, "writeObjectMethod", Method.class), - oscLookup.findVarHandle(ObjectStreamClass.class, "readObjectMethod", Method.class), - oscLookup.findVarHandle(ObjectStreamClass.class, "readObjectNoDataMethod", Method.class), - oscLookup.findVarHandle(ObjectStreamClass.class, "writeReplaceMethod", Method.class), - oscLookup.findVarHandle(ObjectStreamClass.class, "readResolveMethod", Method.class)); - } catch (Throwable ignored) { - return null; - } - } - private static boolean canOpen(String className) { try { return canOpen(Class.forName(className)); @@ -259,54 +189,10 @@ private StringHandles(VarHandle value, VarHandle coder, VarHandle count, VarHand } } - private static class StreamHandles { - private final VarHandle basBuf; - private final VarHandle basCount; - private final VarHandle bisBuf; - private final VarHandle bisPos; - private final VarHandle bisCount; - - private StreamHandles( - VarHandle basBuf, - VarHandle basCount, - VarHandle bisBuf, - VarHandle bisPos, - VarHandle bisCount) { - this.basBuf = basBuf; - this.basCount = basCount; - this.bisBuf = bisBuf; - this.bisPos = bisPos; - this.bisCount = bisCount; - } - } - - private static class ObjectStreamHandles { - private final VarHandle writeObjectMethod; - private final VarHandle readObjectMethod; - private final VarHandle readObjectNoDataMethod; - private final VarHandle writeReplaceMethod; - private final VarHandle readResolveMethod; - - private ObjectStreamHandles( - VarHandle writeObjectMethod, - VarHandle readObjectMethod, - VarHandle readObjectNoDataMethod, - VarHandle writeReplaceMethod, - VarHandle readResolveMethod) { - this.writeObjectMethod = writeObjectMethod; - this.readObjectMethod = readObjectMethod; - this.readObjectNoDataMethod = readObjectNoDataMethod; - this.writeReplaceMethod = writeReplaceMethod; - this.readResolveMethod = readResolveMethod; - } - } - - // The root native-image configuration names these root lazy helpers. Keep same-named JDK25 - // shadows so multi-release class lookup does not fall back to the root Unsafe offset helpers. + // The root native-image configuration names this root lazy helper. Keep a same-named JDK25 + // shadow so multi-release class lookup does not fall back to the root Unsafe offset helper. private static class StringCoderField {} - private static class ByteArrayStreamFields {} - public static Object getStringValue(String value) { checkStringAccess("String.value"); return STRING_VALUE_HANDLE.get(value); @@ -541,86 +427,93 @@ private static MethodHandle getJavaStringZeroCopyCtrHandle() { } } - public static void wrap(ByteArrayOutputStream stream, MemoryBuffer buffer) { - Preconditions.checkNotNull(stream); - checkByteArrayStreamAccess("ByteArrayOutputStream"); - byte[] buf = (byte[]) BAS_BUF_HANDLE.get(stream); - int count = (int) BAS_COUNT_HANDLE.get(stream); - buffer.pointTo(buf, 0, buf.length); - buffer.writerIndex(count); - } - - public static void wrap(MemoryBuffer buffer, ByteArrayOutputStream stream) { - Preconditions.checkNotNull(stream); - checkByteArrayStreamAccess("ByteArrayOutputStream"); - byte[] bytes = buffer.getHeapMemory(); - Preconditions.checkNotNull(bytes); - BAS_BUF_HANDLE.set(stream, bytes); - BAS_COUNT_HANDLE.set(stream, buffer.writerIndex()); + private static void checkStringAccess(String target) { + if (!JDK_STRING_FIELD_ACCESS) { + throw new UnsupportedOperationException( + target + + " private access is unavailable; open java.base/java.lang to " + + "org.apache.fory.core,org.apache.fory.format"); + } } - public static void wrap(ByteArrayInputStream stream, MemoryBuffer buffer) { - Preconditions.checkNotNull(stream); - checkByteArrayStreamAccess("ByteArrayInputStream"); - byte[] buf = (byte[]) BIS_BUF_HANDLE.get(stream); - int count = (int) BIS_COUNT_HANDLE.get(stream); - int pos = (int) BIS_POS_HANDLE.get(stream); - buffer.pointTo(buf, 0, count); - buffer.readerIndex(pos); - } + private static class SerializationMethods { + private static final Object REFLECTION_FACTORY; + private static final Method WRITE_OBJECT; + private static final Method READ_OBJECT; + private static final Method READ_OBJECT_NO_DATA; + private static final Method WRITE_REPLACE; + private static final Method READ_RESOLVE; - public static Method getObjectStreamClassWriteObjectMethod(ObjectStreamClass objectStreamClass) { - checkObjectStreamAccess("ObjectStreamClass.writeObjectMethod"); - return (Method) OSC_WRITE_OBJECT_METHOD_HANDLE.get(objectStreamClass); + static { + Object reflectionFactory = null; + Method writeObject = null; + Method readObject = null; + Method readObjectNoData = null; + Method writeReplace = null; + Method readResolve = null; + try { + Class factoryClass = Class.forName("sun.reflect.ReflectionFactory"); + Method getReflectionFactory = factoryClass.getDeclaredMethod("getReflectionFactory"); + reflectionFactory = getReflectionFactory.invoke(null); + writeObject = factoryClass.getDeclaredMethod("writeObjectForSerialization", Class.class); + readObject = factoryClass.getDeclaredMethod("readObjectForSerialization", Class.class); + readObjectNoData = + factoryClass.getDeclaredMethod("readObjectNoDataForSerialization", Class.class); + writeReplace = factoryClass.getDeclaredMethod("writeReplaceForSerialization", Class.class); + readResolve = factoryClass.getDeclaredMethod("readResolveForSerialization", Class.class); + } catch (Throwable e) { + ExceptionUtils.ignore(e); + } + REFLECTION_FACTORY = reflectionFactory; + WRITE_OBJECT = writeObject; + READ_OBJECT = readObject; + READ_OBJECT_NO_DATA = readObjectNoData; + WRITE_REPLACE = writeReplace; + READ_RESOLVE = readResolve; + } } - public static Method getObjectStreamClassReadObjectMethod(ObjectStreamClass objectStreamClass) { - checkObjectStreamAccess("ObjectStreamClass.readObjectMethod"); - return (Method) OSC_READ_OBJECT_METHOD_HANDLE.get(objectStreamClass); + private static Method getSerializationMethod(Class type, Method factoryMethod) { + if (!isSerializationHookLookupAvailable() || factoryMethod == null) { + return null; + } + try { + MethodHandle handle = + (MethodHandle) factoryMethod.invoke(SerializationMethods.REFLECTION_FACTORY, type); + return handle == null ? null : MethodHandles.reflectAs(Method.class, handle); + } catch (Throwable e) { + ExceptionUtils.ignore(e); + return null; + } } - public static Method getObjectStreamClassReadObjectNoDataMethod( - ObjectStreamClass objectStreamClass) { - checkObjectStreamAccess("ObjectStreamClass.readObjectNoDataMethod"); - return (Method) OSC_READ_OBJECT_NO_DATA_METHOD_HANDLE.get(objectStreamClass); + public static Method getSerializationWriteObjectMethod(Class type) { + return getSerializationMethod(type, SerializationMethods.WRITE_OBJECT); } - public static Method getObjectStreamClassWriteReplaceMethod( - ObjectStreamClass objectStreamClass) { - checkObjectStreamAccess("ObjectStreamClass.writeReplaceMethod"); - return (Method) OSC_WRITE_REPLACE_METHOD_HANDLE.get(objectStreamClass); + public static Method getSerializationReadObjectMethod(Class type) { + return getSerializationMethod(type, SerializationMethods.READ_OBJECT); } - public static Method getObjectStreamClassReadResolveMethod(ObjectStreamClass objectStreamClass) { - checkObjectStreamAccess("ObjectStreamClass.readResolveMethod"); - return (Method) OSC_READ_RESOLVE_METHOD_HANDLE.get(objectStreamClass); + public static Method getSerializationReadObjectNoDataMethod(Class type) { + return getSerializationMethod(type, SerializationMethods.READ_OBJECT_NO_DATA); } - private static void checkStringAccess(String target) { - if (!JDK_STRING_FIELD_ACCESS) { - throw new UnsupportedOperationException( - target - + " private access is unavailable; open java.base/java.lang to " - + "org.apache.fory.core,org.apache.fory.format"); - } + public static Method getSerializationWriteReplaceMethod(Class type) { + return getSerializationMethod(type, SerializationMethods.WRITE_REPLACE); } - private static void checkByteArrayStreamAccess(String target) { - if (!JDK_BYTE_ARRAY_STREAM_FIELD_ACCESS) { - throw new UnsupportedOperationException( - target - + " private access is unavailable; open java.base/java.io to " - + "org.apache.fory.core,org.apache.fory.format"); - } + public static Method getSerializationReadResolveMethod(Class type) { + return getSerializationMethod(type, SerializationMethods.READ_RESOLVE); } - private static void checkObjectStreamAccess(String target) { - if (!JDK_OBJECT_STREAM_FIELD_ACCESS) { - throw new UnsupportedOperationException( - target - + " private access is unavailable; open java.base/java.io to " - + "org.apache.fory.core,org.apache.fory.format"); - } + public static boolean isSerializationHookLookupAvailable() { + return SerializationMethods.REFLECTION_FACTORY != null + && SerializationMethods.WRITE_OBJECT != null + && SerializationMethods.READ_OBJECT != null + && SerializationMethods.READ_OBJECT_NO_DATA != null + && SerializationMethods.WRITE_REPLACE != null + && SerializationMethods.READ_RESOLVE != null; } public static T tryMakeFunction( diff --git a/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties b/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties index 99024c974e..57cd8cc996 100644 --- a/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties +++ b/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties @@ -535,7 +535,6 @@ Args=--initialize-at-build-time=org.apache.fory.annotation.ForyField$Dynamic,\ org.apache.fory.util.record.RecordUtils$2,\ org.apache.fory.util.record.RecordUtils$3,\ org.apache.fory.util.record.RecordUtils,\ - org.apache.fory.platform.internal._JDKAccess$ByteArrayStreamFields,\ org.apache.fory.platform.internal._JDKAccess$StringCoderField,\ org.apache.fory.platform.internal._JDKAccess,\ org.apache.fory.platform.internal._Lookup,\ diff --git a/java/fory-core/src/test/java/org/apache/fory/TestUtils.java b/java/fory-core/src/test/java/org/apache/fory/TestUtils.java index 67ed2b8823..22bd3edce5 100644 --- a/java/fory-core/src/test/java/org/apache/fory/TestUtils.java +++ b/java/fory-core/src/test/java/org/apache/fory/TestUtils.java @@ -68,10 +68,6 @@ private static List forkJvmArgs() { args.add("--add-opens=java.base/jdk.internal.reflect=ALL-UNNAMED"); args.add("--add-opens=java.base/java.util=ALL-UNNAMED"); args.add("--add-opens=java.base/java.util.concurrent=ALL-UNNAMED"); - args.add("--add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED"); - args.add("--add-opens=java.base/java.io=ALL-UNNAMED"); - args.add("--add-opens=java.base/java.net=ALL-UNNAMED"); - args.add("--add-opens=java.base/java.math=ALL-UNNAMED"); if (hasInputArg("--sun-misc-unsafe-memory-access=deny")) { args.add("--sun-misc-unsafe-memory-access=deny"); } diff --git a/java/fory-core/src/test/java/org/apache/fory/memory/MemoryBufferTest.java b/java/fory-core/src/test/java/org/apache/fory/memory/MemoryBufferTest.java index 1ed4fbbdaa..58fe730cc1 100644 --- a/java/fory-core/src/test/java/org/apache/fory/memory/MemoryBufferTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/memory/MemoryBufferTest.java @@ -23,7 +23,6 @@ import static org.testng.Assert.assertThrows; import static org.testng.Assert.assertTrue; -import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; @@ -85,29 +84,6 @@ public void testBufferWrite() { assertEquals(buffer.readerIndex(), buffer.writerIndex()); } - @Test - public void testByteArrayStreamWrap() { - if (!MemoryUtils.JDK_BYTE_ARRAY_STREAM_FIELD_ACCESS) { - return; - } - ByteArrayOutputStream outputStream = new ByteArrayOutputStream(8); - outputStream.write(new byte[] {1, 2, 3}, 0, 3); - MemoryBuffer buffer = MemoryUtils.buffer(1); - MemoryUtils.wrap(outputStream, buffer); - assertEquals(buffer.writerIndex(), 3); - assertEquals(buffer.getByte(0), (byte) 1); - buffer.writeByte((byte) 4); - MemoryUtils.wrap(buffer, outputStream); - assertEquals(outputStream.size(), 4); - assertEquals(outputStream.toByteArray(), new byte[] {1, 2, 3, 4}); - - ByteArrayInputStream inputStream = new ByteArrayInputStream(new byte[] {5, 6, 7}); - assertEquals(inputStream.read(), 5); - MemoryUtils.wrap(inputStream, buffer); - assertEquals(buffer.readerIndex(), 1); - assertEquals(buffer.readByte(), (byte) 6); - } - @Test public void testFromDirectByteBufferRejectsHeapBuffer() { assertThrows( diff --git a/java/fory-core/src/test/java/org/apache/fory/serializer/AndroidDynamicFeatureTest.java b/java/fory-core/src/test/java/org/apache/fory/serializer/AndroidDynamicFeatureTest.java index 828289c39e..716381df2b 100644 --- a/java/fory-core/src/test/java/org/apache/fory/serializer/AndroidDynamicFeatureTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/serializer/AndroidDynamicFeatureTest.java @@ -19,7 +19,6 @@ package org.apache.fory.serializer; -import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; @@ -82,7 +81,7 @@ public static void main(String[] args) { LambdaSerializer.STUB_LAMBDA_CLASS == LambdaSerializer.ReplaceStub.class, "Android must not create a runtime lambda stub class"); verifyReflectiveGetter(); - verifyMemoryUtilsStreamWrapGuards(); + verifyJdkInternalFieldAccessDisabled(); verifyXlangUnion(); verifyFory(false); @@ -167,21 +166,10 @@ private static void verifyOutputStreamSerialization(Fory fory) { checkEquals(fory.deserialize(outputStream.toByteArray()), value, "OutputStream round trip"); } - private static void verifyMemoryUtilsStreamWrapGuards() { + private static void verifyJdkInternalFieldAccessDisabled() { check( !MemoryUtils.JDK_INTERNAL_FIELD_ACCESS, "Android must report JDK internal field access unsupported"); - expectUnsupportedAndroidWrap( - () -> MemoryUtils.wrap(new ByteArrayOutputStream(), MemoryUtils.buffer(8)), - "ByteArrayOutputStream direct wrapping"); - expectUnsupportedAndroidWrap( - () -> MemoryUtils.wrap(MemoryUtils.buffer(8), new ByteArrayOutputStream()), - "ByteArrayOutputStream direct wrapping"); - expectUnsupportedAndroidWrap( - () -> - MemoryUtils.wrap( - new ByteArrayInputStream(new byte[] {1, 2, 3}), MemoryUtils.buffer(8)), - "ByteArrayInputStream direct wrapping"); } private static void verifyXlangUnion() { @@ -255,17 +243,6 @@ private static void expectUnsupported(Serializer serializer) { } } - private static void expectUnsupportedAndroidWrap(Runnable operation, String messageFragment) { - try { - operation.run(); - throw new AssertionError("Expected Android unsafe stream wrapping to fail"); - } catch (UnsupportedOperationException expected) { - check( - expected.getMessage().contains(messageFragment), - "Unexpected unsupported message: " + expected.getMessage()); - } - } - private static void check(boolean value, String message) { if (!value) { throw new AssertionError(message); diff --git a/java/fory-core/src/test/java/org/apache/fory/serializer/AndroidJvmRoundTripTest.java b/java/fory-core/src/test/java/org/apache/fory/serializer/AndroidJvmRoundTripTest.java index a003b55c00..9b8f2868b5 100644 --- a/java/fory-core/src/test/java/org/apache/fory/serializer/AndroidJvmRoundTripTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/serializer/AndroidJvmRoundTripTest.java @@ -158,7 +158,6 @@ private static void addAddOpens(ArrayList command) { if (System.getProperty("java.specification.version").startsWith("1.")) { return; } - addAddOpens(command, "java.base/java.io=ALL-UNNAMED"); addAddOpens(command, "java.base/java.lang=ALL-UNNAMED"); addAddOpens(command, "java.base/java.lang.invoke=ALL-UNNAMED"); addAddOpens(command, "java.base/java.lang.reflect=ALL-UNNAMED"); diff --git a/java/fory-format/pom.xml b/java/fory-format/pom.xml index 635b13f40c..c54ed44b2d 100644 --- a/java/fory-format/pom.xml +++ b/java/fory-format/pom.xml @@ -112,16 +112,6 @@ - - org.apache.maven.plugins - maven-surefire-plugin - - - ${argLine} - --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED - - - diff --git a/scala/build.sbt b/scala/build.sbt index a3c264b8f1..63407932f2 100644 --- a/scala/build.sbt +++ b/scala/build.sbt @@ -55,10 +55,6 @@ Test / javaOptions ++= Seq( "--add-opens=java.base/jdk.internal.reflect=ALL-UNNAMED", "--add-opens=java.base/java.util=ALL-UNNAMED", "--add-opens=java.base/java.util.concurrent=ALL-UNNAMED", - "--add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED", - "--add-opens=java.base/java.io=ALL-UNNAMED", - "--add-opens=java.base/java.net=ALL-UNNAMED", - "--add-opens=java.base/java.math=ALL-UNNAMED", ) lazy val writeTestClasspath = taskKey[File]("Writes the Scala test runtime classpath") From 844009e7754da7f7a88ce7736557c175fbdea53b Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Mon, 1 Jun 2026 23:53:36 +0800 Subject: [PATCH 36/69] feat(java): remove unsafe requirement on jdk25 --- .agents/languages/java.md | 5 + .github/workflows/ci.yml | 2 +- ci/run_ci.py | 1 + ci/run_ci.sh | 71 +- ci/tasks/java.py | 76 +- docs/guide/java/native-serialization.md | 51 + docs/guide/java/troubleshooting.md | 37 +- integration_tests/graalvm_tests/pom.xml | 10 - .../jdk_compatibility_tests/pom.xml | 18 +- .../fory/integration_tests/ForyTest.java | 45 + integration_tests/jpms_tests/pom.xml | 20 +- .../model/PrivateFieldBean.java | 8 +- .../JpmsFieldAccessorTest.java | 13 + .../processing/ForyStructProcessor.java | 21 +- .../annotation/processing/SourceField.java | 3 + .../StaticSerializerSourceWriter.java | 367 +++++++- .../processing/ForyStructProcessorTest.java | 101 ++ java/fory-core/pom.xml | 126 ++- .../apache/fory/AbstractThreadSafeFory.java | 7 + .../main/java/org/apache/fory/BaseFory.java | 12 + .../src/main/java/org/apache/fory/Fory.java | 17 + .../fory/annotation/ForyConstructor.java | 41 + .../apache/fory/builder/AccessorHelper.java | 2 +- .../fory/builder/BaseObjectCodecBuilder.java | 33 +- .../org/apache/fory/builder/CodecBuilder.java | 118 ++- .../fory/builder/CompatibleCodecBuilder.java | 5 +- .../fory/builder/ObjectCodecBuilder.java | 22 +- .../fory/builder/UnsafeCodegenSupport.java | 84 ++ .../org/apache/fory/context/CopyContext.java | 23 +- .../org/apache/fory/context/MapRefReader.java | 86 +- .../org/apache/fory/context/ReadContext.java | 30 - .../org/apache/fory/context/RefReader.java | 29 - .../org/apache/fory/memory/MemoryUtils.java | 4 +- .../fory/platform/internal/_JDKAccess.java | 26 +- .../apache/fory/reflect/FieldAccessor.java | 885 +----------------- .../fory/reflect/FieldAccessorFactory.java | 331 +++++++ .../fory/reflect/FieldAccessorStrategy.java | 648 +++++++++++++ .../fory/reflect/ObjectCreatorRegistry.java | 51 + .../apache/fory/reflect/ObjectCreators.java | 381 +++----- .../fory/reflect/ReflectionFieldAccessor.java | 2 +- .../fory/reflect/UnsafeObjectAllocator.java | 47 + .../apache/fory/resolver/ClassResolver.java | 29 +- .../apache/fory/resolver/SharedRegistry.java | 6 + .../apache/fory/resolver/TypeResolver.java | 12 + .../serializer/AbstractObjectSerializer.java | 147 ++- .../CompatibleLayerSerializerBase.java | 33 + .../fory/serializer/CompatibleSerializer.java | 3 +- .../fory/serializer/ExceptionSerializers.java | 20 +- .../fory/serializer/ObjectSerializer.java | 3 +- .../serializer/ObjectStreamSerializer.java | 298 +++--- .../serializer/ReplaceResolveSerializer.java | 19 +- .../apache/fory/serializer/Serializers.java | 2 +- .../StaticGeneratedStructSerializer.java | 121 ++- .../src/main/java25/module-info.java | 62 ++ .../fory/builder/UnsafeCodegenSupport.java | 49 + .../fory/platform/internal/DefineClass.java | 10 +- .../fory/platform/internal/_JDKAccess.java | 155 +-- .../fory/platform/internal/_Lookup.java | 57 +- ...cessor.java => FieldAccessorStrategy.java} | 777 ++------------- .../fory/reflect/UnsafeObjectAllocator.java | 86 ++ .../test/java/org/apache/fory/ForyTest.java | 14 +- .../test/java/org/apache/fory/TestUtils.java | 49 +- .../builder/BuilderUnsafeClassGraphTest.java | 104 ++ .../pkgprivate/PackagePrivateMapKeyTest.java | 6 +- .../fory/reflect/FieldAccessorTest.java | 12 +- .../fory/reflect/ObjectCreatorsTest.java | 14 +- .../fory/reflect/ReflectionUtilsTest.java | 5 +- .../serializer/AndroidJvmRoundTripTest.java | 1 - .../fory/serializer/ArraySerializersTest.java | 8 +- .../fory/serializer/ObjectSerializerTest.java | 63 +- .../collection/CollectionSerializersTest.java | 6 +- .../collection/MapSerializersTest.java | 10 +- java/pom.xml | 36 +- .../kotlin/ksp/ForyKotlinSymbolProcessor.kt | 140 ++- .../ksp/KotlinSerializerSourceWriter.kt | 284 +++++- .../org/apache/fory/kotlin/ksp/Model.kt | 1 + .../kotlin/ksp/ProcessorValidationTest.kt | 68 ++ .../fory/kotlin/xlang/KotlinXlangPeer.kt | 21 + scala/build.sbt | 5 - 79 files changed, 3927 insertions(+), 2668 deletions(-) create mode 100644 java/fory-core/src/main/java/org/apache/fory/annotation/ForyConstructor.java create mode 100644 java/fory-core/src/main/java/org/apache/fory/builder/UnsafeCodegenSupport.java create mode 100644 java/fory-core/src/main/java/org/apache/fory/reflect/FieldAccessorFactory.java create mode 100644 java/fory-core/src/main/java/org/apache/fory/reflect/FieldAccessorStrategy.java create mode 100644 java/fory-core/src/main/java/org/apache/fory/reflect/ObjectCreatorRegistry.java create mode 100644 java/fory-core/src/main/java/org/apache/fory/reflect/UnsafeObjectAllocator.java create mode 100644 java/fory-core/src/main/java25/module-info.java create mode 100644 java/fory-core/src/main/java25/org/apache/fory/builder/UnsafeCodegenSupport.java rename java/fory-core/src/main/java25/org/apache/fory/reflect/{FieldAccessor.java => FieldAccessorStrategy.java} (51%) create mode 100644 java/fory-core/src/main/java25/org/apache/fory/reflect/UnsafeObjectAllocator.java create mode 100644 java/fory-core/src/test/java/org/apache/fory/builder/BuilderUnsafeClassGraphTest.java diff --git a/.agents/languages/java.md b/.agents/languages/java.md index 582e032248..e542544ba8 100644 --- a/.agents/languages/java.md +++ b/.agents/languages/java.md @@ -52,6 +52,11 @@ Load this file when changing anything under `java/` or when Java drives a cross- - Keep GraalVM feature code as a thin metadata/registration layer. Build time should publish metadata needed for runtime reconstruction, not retain concrete generated or user serializer instances in the image heap. - If changes touch GraalVM bootstrap, serializer retention, native-image metadata, or `ObjectStreamSerializer` GraalVM behavior, verify the native-image build and run the produced binary; a plain Java compile is insufficient. - Put latest-JDK or virtual-thread tests in the latest-JDK test modules with the matching compiler/profile floor, and centralize runtime-version probing in existing compatibility utilities. +- For JDK25+ zero-Unsafe work, preserve serializer-family selection by type and configuration. Do not switch a type from `ObjectStreamSerializer` or another Fory serializer family to `JavaSerializer`, a JDK stream fallback, or any broad `java.* Serializable` fallback by JDK version or no-arg-constructor shape. +- Keep JDK25+ unsafe-removal implementation invariants in agent/design docs and tests, not user guides. User guides should document user actions such as `--sun-misc-unsafe-memory-access=deny`, `java.base/java.lang.invoke` opens, JDK26+ `--enable-final-field-mutation`, package opens for user named modules, and public APIs such as `@ForyConstructor` and `BaseFory.registerConstructor(...)`; do not expose internal serializer names, owner-model rationale, or avoided fallback strategies there. +- For JDK25+ zero-Unsafe final-field behavior, distinguish JDK25 from JDK26+: JDK25 has no final-field mutation flag requirement, while JDK26+ requires `--enable-final-field-mutation` for post-construction final-field writes. +- For JDK25+ object creation, do not use `sun.reflect.ReflectionFactory` or `jdk.internal.reflect.ReflectionFactory`. Serializable classes without a no-arg constructor may use `ObjectStreamClass.newInstance` through the trusted lookup owner; non-Serializable classes without a no-arg constructor require `@ForyConstructor`, `BaseFory.registerConstructor(...)`, record construction, or a custom serializer. +- For JDK25+ CI, do not run core runtime tests from raw Maven reactor `target/classes`. Those classes bypass `META-INF/versions/25` and exercise the JDK8-24 root implementation. Build/install the multi-release artifact first, then test the packaged artifact through the classpath integration suite and the JPMS module-path suite. ## Key Modules diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 03b7b2b5a8..d9c794f51c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -169,7 +169,7 @@ jobs: MY_VAR: "PATH" strategy: matrix: - java-version: ${{ fromJSON(needs.changes.outputs.java_code == 'true' && '["8","11","17","21","25"]' || '["8"]') }} + java-version: ${{ fromJSON(needs.changes.outputs.java_code == 'true' && '["8","11","17","21","25","26"]' || '["8"]') }} steps: - uses: actions/checkout@v5 - name: Set up JDK ${{ matrix.java-version }} diff --git a/ci/run_ci.py b/ci/run_ci.py index 334ad7c77c..12192dcbd4 100644 --- a/ci/run_ci.py +++ b/ci/run_ci.py @@ -315,6 +315,7 @@ def parse_args(): "17", "21", "25", + "26", "windows_java21", "integration_tests", "graalvm", diff --git a/ci/run_ci.sh b/ci/run_ci.sh index a742161d8e..cf8beb52ca 100755 --- a/ci/run_ci.sh +++ b/ci/run_ci.sh @@ -86,7 +86,7 @@ graalvm_test() { java_major=$(echo "$java_version" | cut -d. -f1) fi if [[ "$java_major" -ge 25 ]]; then - export JDK_JAVA_OPTIONS="$(jdk25_deny_options) $(jdk25_javac_options)" + export JDK_JAVA_OPTIONS="$(jdk25_javac_options)" else unset JDK_JAVA_OPTIONS fi @@ -101,15 +101,24 @@ graalvm_test() { echo "Execute graalvm tests succeed!" } -jdk25_deny_options() { - local fory_open_targets="ALL-UNNAMED,org.apache.fory.core,org.apache.fory.format" +jdk25_access_options() { + local fory_open_targets="${1:-ALL-UNNAMED}" printf "%s" "--sun-misc-unsafe-memory-access=deny" - printf " %s" "--add-opens=java.base/java.lang=${fory_open_targets}" printf " %s" "--add-opens=java.base/java.lang.invoke=${fory_open_targets}" - printf " %s" "--add-opens=java.base/java.lang.reflect=${fory_open_targets}" - printf " %s" "--add-opens=java.base/jdk.internal.reflect=${fory_open_targets}" - printf " %s" "--add-opens=java.base/java.util=${fory_open_targets}" - printf " %s" "--add-opens=java.base/java.util.concurrent=${fory_open_targets}" +} + +jdk26_final_field_options() { + local fory_modules="${1:-ALL-UNNAMED}" + printf "%s" "--enable-final-field-mutation=${fory_modules}" +} + +jdk25_plus_options() { + local java_major="$1" + local fory_targets="${2:-ALL-UNNAMED}" + printf "%s" "$(jdk25_access_options "$fory_targets")" + if [[ "$java_major" -ge 26 ]]; then + printf " %s" "$(jdk26_final_field_options "$fory_targets")" + fi } jdk25_javac_options() { @@ -130,8 +139,13 @@ use_jdk() { local jdk="$1" export JAVA_HOME="$ROOT/$jdk" export PATH=$JAVA_HOME/bin:$PATH - if [[ "$jdk" == zulu25* ]]; then - export JDK_JAVA_OPTIONS="$(jdk25_deny_options) $(jdk25_javac_options)" + if [[ "$jdk" =~ zulu([0-9]+) ]]; then + local java_major="${BASH_REMATCH[1]}" + if [[ "$java_major" -ge 25 ]]; then + export JDK_JAVA_OPTIONS="$(jdk25_plus_options "$java_major") $(jdk25_javac_options)" + else + unset JDK_JAVA_OPTIONS + fi else unset JDK_JAVA_OPTIONS fi @@ -147,11 +161,13 @@ install_jdk25_fory_artifacts() { [[ -n "${JDK_JAVA_OPTIONS+x}" ]] && had_jdk_java_options=1 use_jdk "zulu25.30.17-ca-jdk25.0.1-linux_x64" + export JDK_JAVA_OPTIONS="$(jdk25_javac_options)" cd "$ROOT"/java mvn -T10 -B --no-transfer-progress clean install -DskipTests -Dmaven.compiler.parameters=true -pl '!:fory-testsuite' echo "Verify JDK25 benchmark multi-release jar" cd "$ROOT"/benchmarks/java mvn -T10 -B --no-transfer-progress -Pjmh -DskipTests install + unset JDK_JAVA_OPTIONS echo "Verify JPMS tests on JDK25" cd "$ROOT"/integration_tests/jpms_tests mvn -T10 -B --no-transfer-progress clean test @@ -203,21 +219,41 @@ jdk17_plus_tests() { fi JDK_JAVA_OPTIONS="--add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED" if [[ "$java_major" -ge 25 ]]; then - JDK_JAVA_OPTIONS="$JDK_JAVA_OPTIONS $(jdk25_deny_options) $(jdk25_javac_options)" + JDK_JAVA_OPTIONS="$JDK_JAVA_OPTIONS $(jdk25_javac_options)" fi export JDK_JAVA_OPTIONS echo "Executing fory java tests" cd "$ROOT/java" set +e - jdk25_test_classpath=() if [[ "$java_major" -ge 25 ]]; then - jdk25_test_classpath=(-Dfory.jdk25.test.classpath=true -Dmaven.compiler.parameters=true) + # JDK25+ must be tested from the packaged multi-release artifact. Raw + # reactor test classes bypass META-INF/versions/25 and exercise the + # JDK8-24 root implementation instead. + mvn -T10 --batch-mode --no-transfer-progress clean install -DskipTests -Dmaven.compiler.parameters=true + else + mvn -T10 --batch-mode --no-transfer-progress install fi - mvn -T10 --batch-mode --no-transfer-progress install "${jdk25_test_classpath[@]}" testcode=$? if [[ $testcode -ne 0 ]]; then exit $testcode fi + if [[ "$java_major" -ge 25 ]]; then + unset JDK_JAVA_OPTIONS + echo "Executing JDK${java_major} packaged classpath tests" + cd "$ROOT/integration_tests/jdk_compatibility_tests" + mvn -T10 --batch-mode --no-transfer-progress clean test + testcode=$? + if [[ $testcode -ne 0 ]]; then + exit $testcode + fi + echo "Executing JDK${java_major} JPMS tests" + cd "$ROOT/integration_tests/jpms_tests" + mvn -T10 --batch-mode --no-transfer-progress clean test + testcode=$? + if [[ $testcode -ne 0 ]]; then + exit $testcode + fi + fi echo "Executing fory java tests succeeds" } @@ -270,7 +306,7 @@ case $1 in echo "Executing fory java tests" cd "$ROOT/java" set +e - mvn -T16 --batch-mode --no-transfer-progress test + mvn -T16 --batch-mode --no-transfer-progress clean test testcode=$? if [[ $testcode -ne 0 ]]; then exit $testcode @@ -282,7 +318,7 @@ case $1 in echo "Executing fory java tests" cd "$ROOT/java" set +e - mvn -T16 --batch-mode --no-transfer-progress test + mvn -T16 --batch-mode --no-transfer-progress clean test testcode=$? if [[ $testcode -ne 0 ]]; then exit $testcode @@ -298,6 +334,9 @@ case $1 in java25) jdk17_plus_tests ;; + java26) + jdk17_plus_tests + ;; kotlin) kotlin_tests ;; diff --git a/ci/tasks/java.py b/ci/tasks/java.py index 2aca7ad071..7784e1b488 100644 --- a/ci/tasks/java.py +++ b/ci/tasks/java.py @@ -56,6 +56,7 @@ def get_jdk_major_version(): "21": "zulu21.28.85-ca-jdk21.0.0-linux_x64", "24": "zulu24.32.13-ca-fx-jdk24.0.2-linux_x64", "25": "zulu25.30.17-ca-jdk25.0.1-linux_x64", + "26": "zulu26.28.89-ca-jdk26.0.0-linux_x64", } @@ -76,23 +77,24 @@ def install_jdks(): logging.info("JDKs downloaded and installed successfully") -def jdk25_deny_options(): - fory_open_targets = "ALL-UNNAMED,org.apache.fory.core,org.apache.fory.format" +def jdk25_access_options(fory_targets="ALL-UNNAMED"): return [ "--sun-misc-unsafe-memory-access=deny", - f"--add-opens=java.base/java.lang={fory_open_targets}", - f"--add-opens=java.base/java.lang.invoke={fory_open_targets}", - f"--add-opens=java.base/java.lang.reflect={fory_open_targets}", - f"--add-opens=java.base/jdk.internal.reflect={fory_open_targets}", - f"--add-opens=java.base/java.util={fory_open_targets}", - f"--add-opens=java.base/java.util.concurrent={fory_open_targets}", - f"--add-opens=java.base/java.util.concurrent.atomic={fory_open_targets}", - f"--add-opens=java.base/java.io={fory_open_targets}", - f"--add-opens=java.base/java.net={fory_open_targets}", - f"--add-opens=java.base/java.math={fory_open_targets}", + f"--add-opens=java.base/java.lang.invoke={fory_targets}", ] +def jdk26_final_field_options(fory_targets="ALL-UNNAMED"): + return [f"--enable-final-field-mutation={fory_targets}"] + + +def jdk25_plus_options(java_version, fory_targets="ALL-UNNAMED"): + options = jdk25_access_options(fory_targets) + if int(java_version) >= 26: + options.extend(jdk26_final_field_options(fory_targets)) + return options + + def jdk25_javac_options(): return [ "--add-opens=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED", @@ -110,9 +112,9 @@ def jdk25_javac_options(): def set_jdk_options(java_version): - if java_version == "25": + if int(java_version) >= 25: os.environ["JDK_JAVA_OPTIONS"] = " ".join( - jdk25_deny_options() + jdk25_javac_options() + jdk25_plus_options(java_version) + jdk25_javac_options() ) else: os.environ.pop("JDK_JAVA_OPTIONS", None) @@ -145,6 +147,7 @@ def install_jdk25_fory_artifacts(): env = save_java_env() try: use_jdk("25") + os.environ["JDK_JAVA_OPTIONS"] = " ".join(jdk25_javac_options()) common.cd_project_subdir("java") common.exec_cmd( "mvn -T10 -B --no-transfer-progress clean install -DskipTests " @@ -154,6 +157,7 @@ def install_jdk25_fory_artifacts(): common.cd_project_subdir("benchmarks/java") common.exec_cmd("mvn -T10 -B --no-transfer-progress -Pjmh -DskipTests install") logging.info("Verify JPMS tests on JDK25") + os.environ.pop("JDK_JAVA_OPTIONS", None) common.cd_project_subdir("integration_tests/jpms_tests") common.exec_cmd("mvn -T10 -B --no-transfer-progress clean test") finally: @@ -235,7 +239,7 @@ def run_java8(): logging.info("Executing fory java tests with Java 8") install_jdks() common.cd_project_subdir("java") - common.exec_cmd("mvn -T16 --batch-mode --no-transfer-progress test") + common.exec_cmd("mvn -T16 --batch-mode --no-transfer-progress clean test") logging.info("Executing fory java tests succeeds") @@ -243,7 +247,7 @@ def run_java11(): """Run Java 11 tests.""" logging.info("Executing fory java tests with Java 11") common.cd_project_subdir("java") - common.exec_cmd("mvn -T16 --batch-mode --no-transfer-progress test") + common.exec_cmd("mvn -T16 --batch-mode --no-transfer-progress clean test") logging.info("Executing fory java tests succeeds") @@ -251,25 +255,31 @@ def run_jdk17_plus(java_version="17"): """Run Java 17+ tests.""" logging.info(f"Executing fory java tests with Java {java_version}") common.exec_cmd("java -version") - jdk_options = [] - if java_version == "25": - jdk_options.extend(jdk25_deny_options()) + jdk_options = [ + "--add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED" + ] + if int(java_version) >= 25: jdk_options.extend(jdk25_javac_options()) - else: - jdk_options.append( - "--add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED" - ) os.environ["JDK_JAVA_OPTIONS"] = " ".join(jdk_options) common.cd_project_subdir("java") - jdk25_test_classpath = "" - if java_version == "25": - jdk25_test_classpath = ( - " -Dfory.jdk25.test.classpath=true -Dmaven.compiler.parameters=true" + if int(java_version) >= 25: + # JDK25+ must be tested from the packaged multi-release artifact. Raw + # reactor test classes bypass META-INF/versions/25 and exercise the + # JDK8-24 root implementation instead. + common.exec_cmd( + "mvn -T10 --batch-mode --no-transfer-progress clean install " + "-DskipTests -Dmaven.compiler.parameters=true" ) - common.exec_cmd( - f"mvn -T10 --batch-mode --no-transfer-progress install{jdk25_test_classpath}" - ) + os.environ.pop("JDK_JAVA_OPTIONS", None) + logging.info(f"Executing JDK{java_version} packaged classpath tests") + common.cd_project_subdir("integration_tests/jdk_compatibility_tests") + common.exec_cmd("mvn -T10 --batch-mode --no-transfer-progress clean test") + logging.info(f"Executing JDK{java_version} JPMS tests") + common.cd_project_subdir("integration_tests/jpms_tests") + common.exec_cmd("mvn -T10 --batch-mode --no-transfer-progress clean test") + else: + common.exec_cmd("mvn -T10 --batch-mode --no-transfer-progress install") logging.info("Executing fory java tests succeeds") @@ -347,9 +357,7 @@ def run_graalvm_test(): logging.info("Start GraalVM tests") java_major = get_jdk_major_version() if java_major is not None and java_major >= 25: - os.environ["JDK_JAVA_OPTIONS"] = " ".join( - jdk25_deny_options() + jdk25_javac_options() - ) + os.environ["JDK_JAVA_OPTIONS"] = " ".join(jdk25_javac_options()) else: os.environ.pop("JDK_JAVA_OPTIONS", None) @@ -411,6 +419,8 @@ def run(version=None, release=False, install_jdks=False, install_fory=False): run_jdk17_plus("21") elif version == "25": run_jdk17_plus("25") + elif version == "26": + run_jdk17_plus("26") elif version == "windows_java21": run_windows_java21() elif version == "integration_tests": diff --git a/docs/guide/java/native-serialization.md b/docs/guide/java/native-serialization.md index 7dd6fe58dc..d13d73b8d4 100644 --- a/docs/guide/java/native-serialization.md +++ b/docs/guide/java/native-serialization.md @@ -162,6 +162,54 @@ For ordinary application classes, Fory can use generated serializers and avoid J serialization-compatible path; prefer a Fory custom serializer for hot classes when the hook-based path is too expensive. +## Final Fields And Constructors + +Records are deserialized through their canonical constructor. Ordinary classes use Fory's normal +object-creation path and field setting unless you provide an explicit constructor mapping. + +Use `@ForyConstructor` when a constructor should receive serialized field values: + +```java +import org.apache.fory.annotation.ForyConstructor; + +public final class User { + private final String name; + private final int age; + + @ForyConstructor({"name", "age"}) + public User(String name, int age) { + this.name = name; + this.age = age; + } +} +``` + +For third-party classes that cannot be annotated, register the constructor during runtime setup: + +```java +import java.lang.reflect.Constructor; + +Constructor constructor = User.class.getDeclaredConstructor(String.class, int.class); +fory.registerConstructor(User.class, constructor, "name", "age"); +``` + +The field names are the binding contract. For ordinary classes, Fory does not infer constructor +bindings from Java parameter names, `-parameters`, or `@ConstructorProperties`. + +When no explicit constructor mapping is provided, normal classes with final fields use Fory's normal +object creation and field setting. On JDK25+ with Unsafe memory access denied, Fory reports an error +if the class cannot be created by supported Java mechanisms. Use `@ForyConstructor`, +`registerConstructor(...)`, a record canonical constructor, or a custom serializer for those classes. +Use the `java.base/java.lang.invoke` open shown in troubleshooting for supported JDK25+ access paths. +On JDK26+, enable final-field mutation for the Fory runtime module, or for `ALL-UNNAMED` when Fory is +loaded from the classpath. JDK25 does not have the final-field mutation flag. See +[Troubleshooting](troubleshooting.md#jdk25-zero-unsafe-mode-and-module-opens) for the required JVM +flags. + +Constructor-bound objects cannot receive a constructor argument that refers directly to the same +object under construction. Model those cycles through non-constructor fields or use a custom +serializer. + ## JDK Serialization Hooks Java native mode supports the JDK serialization hooks that are part of many existing Java object @@ -172,6 +220,9 @@ models: - `readObjectNoData` - `Externalizable` +Fory native serialization remains stable across supported JDK versions when writers and readers use +the same Fory version and runtime configuration. + ```java import java.io.IOException; import java.io.ObjectInputStream; diff --git a/docs/guide/java/troubleshooting.md b/docs/guide/java/troubleshooting.md index 517af28ce5..9dd2e345cb 100644 --- a/docs/guide/java/troubleshooting.md +++ b/docs/guide/java/troubleshooting.md @@ -157,30 +157,25 @@ memory access becomes the default, start the JVM with: --sun-misc-unsafe-memory-access=deny ``` -If Fory needs private fields in your named module, open the target package to both Java modules. -When any Fory artifact is on the classpath instead of the module path, also include `ALL-UNNAMED`: +If Fory runs from the classpath, including a modular Fory jar placed on the classpath, open +`java.base/java.lang.invoke` to the unnamed module: ```bash ---add-opens=/=ALL-UNNAMED,org.apache.fory.core,org.apache.fory.format +--add-opens=java.base/java.lang.invoke=ALL-UNNAMED ``` -Some optimized serializers also need JDK-private packages. For each package in the table, open the -owning JDK module/package to `org.apache.fory.core` and `org.apache.fory.format`; include -`ALL-UNNAMED` too when any Fory artifact is on the classpath. Add only the opens needed by the paths -used in your process: +If Fory runs as named modules on the module path, open `java.base/java.lang.invoke` to the Fory core +module: -| Path | Required opens | -| --------------------------------------------------------------- | --------------------------------------------------------------- | -| String fast paths and throwable fields | `java.base/java.lang` | -| Serialized lambdas | `java.base/java.lang.invoke` | -| Reflection-based object construction | `java.base/java.lang.reflect`, `java.base/jdk.internal.reflect` | -| Collection wrappers, sublists, `EnumMap`, and `StringTokenizer` | `java.base/java.util` | -| Blocking queue capacity serializers | `java.base/java.util.concurrent` | -| Proxy serializers | `java.base/java.lang.reflect` | +```bash +--add-opens=java.base/java.lang.invoke=org.apache.fory.core +``` + +If this open is missing, Fory reports an error that names `java.base/java.lang.invoke`. -Normal classes with final instance fields require final-field mutation to be enabled for the module -that contains Fory's mutating code when Unsafe allocation is denied. Use the Fory module name on the -module path: +On JDK26 and later, normal classes with final instance fields require final-field mutation to be +enabled for the module that contains Fory's mutating code when Unsafe allocation is denied. Use the +Fory module name on the module path: ```bash --enable-final-field-mutation=org.apache.fory.core @@ -192,8 +187,10 @@ Use `ALL-UNNAMED` when running Fory on the classpath: --enable-final-field-mutation=ALL-UNNAMED ``` -Fory restores those final fields through method-handle based access. Non-final fields can still be -restored through generated direct field assignment where available. +Fory can restore those final fields when final-field mutation is enabled. JDK25 has no +`--enable-final-field-mutation` option, so no final-field mutation flag is needed on JDK25. Named +application modules that contain private fields still need to open the application package to +`org.apache.fory.core`. The vectorized Arrow APIs in `fory-format` depend on Apache Arrow's memory layer. With the current Arrow dependency, those APIs are unavailable when `--sun-misc-unsafe-memory-access=deny` is set diff --git a/integration_tests/graalvm_tests/pom.xml b/integration_tests/graalvm_tests/pom.xml index c2be9cd979..acb6ebb4da 100644 --- a/integration_tests/graalvm_tests/pom.xml +++ b/integration_tests/graalvm_tests/pom.xml @@ -176,12 +176,7 @@ -H:+UnlockExperimentalVMOptions - -J--add-opens=java.base/java.lang=ALL-UNNAMED -J--add-opens=java.base/java.lang.invoke=ALL-UNNAMED - -J--add-opens=java.base/java.lang.reflect=ALL-UNNAMED - -J--add-opens=java.base/jdk.internal.reflect=ALL-UNNAMED - -J--add-opens=java.base/java.util=ALL-UNNAMED - -J--add-opens=java.base/java.util.concurrent=ALL-UNNAMED @@ -199,12 +194,7 @@ ${java.home}/bin/java - --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED - --add-opens=java.base/java.lang.reflect=ALL-UNNAMED - --add-opens=java.base/jdk.internal.reflect=ALL-UNNAMED - --add-opens=java.base/java.util=ALL-UNNAMED - --add-opens=java.base/java.util.concurrent=ALL-UNNAMED -classpath ${mainClass} diff --git a/integration_tests/jdk_compatibility_tests/pom.xml b/integration_tests/jdk_compatibility_tests/pom.xml index 6dd1850b8d..d5df74e988 100644 --- a/integration_tests/jdk_compatibility_tests/pom.xml +++ b/integration_tests/jdk_compatibility_tests/pom.xml @@ -35,6 +35,7 @@ 8 8 UTF-8 + @@ -95,18 +96,25 @@ --sun-misc-unsafe-memory-access=deny - --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED - --add-opens=java.base/java.lang.reflect=ALL-UNNAMED - --add-opens=java.base/jdk.internal.reflect=ALL-UNNAMED - --add-opens=java.base/java.util=ALL-UNNAMED - --add-opens=java.base/java.util.concurrent=ALL-UNNAMED + ${fory.final.field.mutation.arg} + + jdk26-and-higher + + [26,) + + + + --enable-final-field-mutation=ALL-UNNAMED + + + diff --git a/integration_tests/jdk_compatibility_tests/src/test/java/org/apache/fory/integration_tests/ForyTest.java b/integration_tests/jdk_compatibility_tests/src/test/java/org/apache/fory/integration_tests/ForyTest.java index 19a2cb7934..527988d2bc 100644 --- a/integration_tests/jdk_compatibility_tests/src/test/java/org/apache/fory/integration_tests/ForyTest.java +++ b/integration_tests/jdk_compatibility_tests/src/test/java/org/apache/fory/integration_tests/ForyTest.java @@ -19,9 +19,11 @@ package org.apache.fory.integration_tests; +import java.io.Serializable; import org.apache.fory.Fory; import org.apache.fory.benchmark.data.MediaContent; import org.apache.fory.benchmark.data.Sample; +import org.apache.fory.platform.JdkVersion; import org.testng.Assert; import org.testng.annotations.Test; @@ -44,4 +46,47 @@ public void testSample() { MediaContent mediaContent = (MediaContent) fory.deserialize(data); Assert.assertEquals(mediaContent, object); } + + @Test + public void testClasspathRuntime() throws Exception { + if (JdkVersion.MAJOR_VERSION >= 9) { + Object module = Class.class.getMethod("getModule").invoke(Fory.class); + boolean named = (boolean) module.getClass().getMethod("isNamed").invoke(module); + Assert.assertFalse(named); + } + } + + @Test + public void testFinalFieldBean() { + Fory fory = Fory.builder().withXlang(false).requireClassRegistration(false).build(); + FinalFieldBean value = new FinalFieldBean("amy", 42); + Object deserialized = fory.deserialize(fory.serialize(value)); + Assert.assertEquals(deserialized, value); + } + + static final class FinalFieldBean implements Serializable { + private static final long serialVersionUID = 1L; + + private final String name; + private final int age; + + private FinalFieldBean(String name, int age) { + this.name = name; + this.age = age; + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof FinalFieldBean)) { + return false; + } + FinalFieldBean other = (FinalFieldBean) obj; + return age == other.age && name.equals(other.name); + } + + @Override + public int hashCode() { + return name.hashCode() * 31 + age; + } + } } diff --git a/integration_tests/jpms_tests/pom.xml b/integration_tests/jpms_tests/pom.xml index 5f229a57b8..07ec323c74 100644 --- a/integration_tests/jpms_tests/pom.xml +++ b/integration_tests/jpms_tests/pom.xml @@ -36,6 +36,7 @@ 11 11 UTF-8 + @@ -90,18 +91,25 @@ --sun-misc-unsafe-memory-access=deny - --add-opens=java.base/java.lang=org.apache.fory.core,org.apache.fory.format - --add-opens=java.base/java.lang.invoke=org.apache.fory.core,org.apache.fory.format - --add-opens=java.base/java.lang.reflect=org.apache.fory.core,org.apache.fory.format - --add-opens=java.base/jdk.internal.reflect=org.apache.fory.core,org.apache.fory.format - --add-opens=java.base/java.util=org.apache.fory.core,org.apache.fory.format - --add-opens=java.base/java.util.concurrent=org.apache.fory.core,org.apache.fory.format + --add-opens=java.base/java.lang.invoke=org.apache.fory.core + ${fory.final.field.mutation.arg} + + jdk26-and-higher + + [26,) + + + + --enable-final-field-mutation=org.apache.fory.core + + + diff --git a/integration_tests/jpms_tests/src/main/java/org/apache/fory/integration_tests/model/PrivateFieldBean.java b/integration_tests/jpms_tests/src/main/java/org/apache/fory/integration_tests/model/PrivateFieldBean.java index 8b0c9e326d..036d405fac 100644 --- a/integration_tests/jpms_tests/src/main/java/org/apache/fory/integration_tests/model/PrivateFieldBean.java +++ b/integration_tests/jpms_tests/src/main/java/org/apache/fory/integration_tests/model/PrivateFieldBean.java @@ -19,8 +19,12 @@ package org.apache.fory.integration_tests.model; -public final class PrivateFieldBean { - private int value; +import java.io.Serializable; + +public final class PrivateFieldBean implements Serializable { + private static final long serialVersionUID = 1L; + + private final int value; public PrivateFieldBean(int value) { this.value = value; diff --git a/integration_tests/jpms_tests/src/test/java/org/apache/fory/integration_tests/JpmsFieldAccessorTest.java b/integration_tests/jpms_tests/src/test/java/org/apache/fory/integration_tests/JpmsFieldAccessorTest.java index fdfbe0d49e..5eea91bcbb 100644 --- a/integration_tests/jpms_tests/src/test/java/org/apache/fory/integration_tests/JpmsFieldAccessorTest.java +++ b/integration_tests/jpms_tests/src/test/java/org/apache/fory/integration_tests/JpmsFieldAccessorTest.java @@ -39,6 +39,19 @@ public void testPrivateFieldAccess() throws Exception { Assert.assertEquals(bean.value(), 9); } + @Test + public void testPrivateFinalFieldSerialization() { + Fory fory = + Fory.builder() + .withXlang(false) + .withCodegen(false) + .requireClassRegistration(false) + .build(); + PrivateFieldBean result = + (PrivateFieldBean) fory.deserialize(fory.serialize(new PrivateFieldBean(13))); + Assert.assertEquals(result.value(), 13); + } + @Test public void testPublicSerializerInExportedPackage() { Fory fory = Fory.builder().withXlang(false).requireClassRegistration(false).build(); diff --git a/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/ForyStructProcessor.java b/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/ForyStructProcessor.java index 33015c2959..0867d57245 100644 --- a/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/ForyStructProcessor.java +++ b/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/ForyStructProcessor.java @@ -61,7 +61,8 @@ @SupportedAnnotationTypes({ "org.apache.fory.annotation.ForyStruct", - "org.apache.fory.annotation.ForyDebug" + "org.apache.fory.annotation.ForyDebug", + "org.apache.fory.annotation.ForyConstructor" }) public final class ForyStructProcessor extends AbstractProcessor { private static final String ARRAY_TYPE = "org.apache.fory.annotation.ArrayType"; @@ -376,13 +377,6 @@ private SourceField buildField( boolean serialized, SerializerMode mode) { Set modifiers = field.getModifiers(); - if (!record && modifiers.contains(Modifier.FINAL)) { - throw new InvalidStructException( - "Static serializers cannot assign final field " - + field.getSimpleName() - + "; use a record component or mark the field @Ignore/transient", - field); - } ForyFieldMeta foryField = foryField(field); Object fieldTypeTree = typeTree(field); boolean nullable = fieldNullable(field.asType(), fieldTypeTree, mode); @@ -397,6 +391,7 @@ private SourceField buildField( SourceField.AccessKind writeKind; String readAccess; String writeAccess; + boolean finalField = modifiers.contains(Modifier.FINAL); if (record) { readKind = SourceField.AccessKind.METHOD; writeKind = SourceField.AccessKind.METHOD; @@ -409,19 +404,20 @@ private SourceField buildField( writeAccess = readAccess; } else { ExecutableElement getter = findGetter(owner, field, generatedPackage); - ExecutableElement setter = findSetter(owner, field, generatedPackage); - if (getter == null || setter == null) { + ExecutableElement setter = finalField ? null : findSetter(owner, field, generatedPackage); + if (getter == null || (!finalField && setter == null)) { throw new InvalidStructException( "Field " + field.getSimpleName() + " is not directly accessible from the generated serializer. Add accessible " - + "non-private getter/setter methods or mark it @Ignore/transient.", + + (finalField ? "non-private getter" : "non-private getter/setter") + + " methods or mark it @Ignore/transient.", field); } readKind = SourceField.AccessKind.METHOD; writeKind = SourceField.AccessKind.METHOD; readAccess = getter.getSimpleName().toString(); - writeAccess = setter.getSimpleName().toString(); + writeAccess = finalField ? null : setter.getSimpleName().toString(); } return new SourceField( id, @@ -436,6 +432,7 @@ private SourceField buildField( readAccess, writeKind, writeAccess, + finalField, foryField.hasForyField, foryField.id, nullable, diff --git a/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/SourceField.java b/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/SourceField.java index 3d33d37422..e0d7b8a6b1 100644 --- a/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/SourceField.java +++ b/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/SourceField.java @@ -37,6 +37,7 @@ enum AccessKind { final String readAccess; final AccessKind writeAccessKind; final String writeAccess; + final boolean finalField; final boolean hasForyField; final int foryFieldId; final boolean nullable; @@ -57,6 +58,7 @@ enum AccessKind { String readAccess, AccessKind writeAccessKind, String writeAccess, + boolean finalField, boolean hasForyField, int foryFieldId, boolean nullable, @@ -75,6 +77,7 @@ enum AccessKind { this.readAccess = readAccess; this.writeAccessKind = writeAccessKind; this.writeAccess = writeAccess; + this.finalField = finalField; this.hasForyField = hasForyField; this.foryFieldId = foryFieldId; this.nullable = nullable; diff --git a/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/StaticSerializerSourceWriter.java b/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/StaticSerializerSourceWriter.java index ce52ad379d..7eabcda6b2 100644 --- a/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/StaticSerializerSourceWriter.java +++ b/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/StaticSerializerSourceWriter.java @@ -88,6 +88,8 @@ private void writeClassStart() { builder.append(" private final SerializationFieldInfo[] allFields;\n"); builder.append(" private final int[] allFieldIds;\n"); builder.append(" private final SerializationFieldInfo[] fieldsById;\n"); + builder.append(" private final int[] constructorFieldIds;\n"); + builder.append(" private final long[] constructorFieldBits;\n"); builder.append(" private final int classVersionHash;\n"); builder.append(" private final boolean sameSchemaCompatible;\n\n"); } @@ -140,6 +142,8 @@ private void writeConstructors() { builder.append(" this.allFields = null;\n"); builder.append(" this.allFieldIds = null;\n"); builder.append(" this.fieldsById = null;\n"); + builder.append(" this.constructorFieldIds = null;\n"); + builder.append(" this.constructorFieldBits = null;\n"); builder.append(" this.classVersionHash = 0;\n"); builder.append(" this.sameSchemaCompatible = false;\n"); builder.append(" }\n\n"); @@ -171,6 +175,10 @@ private void writeConstructorBody(String fieldGroupsExpression, String sameSchem builder.append(" for (int i = 0; i < allFields.length; i++) {\n"); builder.append(" this.fieldsById[allFieldIds[i]] = allFields[i];\n"); builder.append(" }\n"); + builder.append( + " this.constructorFieldIds = objectCreator.hasConstructorFields() ? buildConstructorFieldIds(DESCRIPTORS) : null;\n"); + builder.append( + " this.constructorFieldBits = buildConstructorFieldBits(DESCRIPTORS.size(), constructorFieldIds);\n"); builder.append( " this.classVersionHash = typeResolver.checkClassVersion() ? computeClassVersionHash(DESCRIPTORS) : 0;\n"); builder.append(" this.sameSchemaCompatible = ").append(sameSchemaExpression).append(";\n"); @@ -235,6 +243,9 @@ private void writeSchemaConsistentRead() { appendRecordConstructorArguments("field"); builder.append(");\n"); } else { + builder.append(" if (constructorFieldIds != null) {\n"); + builder.append(" return readSchemaConsistentConstructor(readContext);\n"); + builder.append(" }\n"); builder.append(" ").append(struct.typeName).append(" value = newBean();\n"); builder.append(" readContext.reference(value);\n"); builder.append(" readFields(readContext, value);\n"); @@ -292,6 +303,7 @@ private void writeReadGroups() { writeReadRecordGroup("", "allFields", "allFieldIds", "readFieldValue"); } else { writeReadBeanGroup("", "allFields", "allFieldIds", "readFieldValue"); + writeConstructorRead(); } } @@ -320,7 +332,11 @@ private void writeReadBeanGroup( for (SourceField field : struct.fields) { builder.append(" case ").append(field.id).append(":\n"); if (canEmitDirectReadField(field)) { - appendDirectRead(field); + if (field.finalField) { + appendFinalDirectRead(field, "value", "fieldInfo"); + } else { + appendDirectRead(field); + } } else { String fieldValueName = "fieldValue" + field.id; if (hasDirectReadField()) { @@ -333,10 +349,17 @@ private void writeReadBeanGroup( } else { fieldValueName = "fieldValue"; } - builder - .append(" ") - .append(field.writeStatement("value", field.castExpression(fieldValueName))) - .append("\n"); + builder.append(" "); + if (field.finalField) { + builder + .append("setGeneratedFieldValue(value, fieldInfo, ") + .append(field.castExpression(fieldValueName)) + .append(");\n"); + } else { + builder + .append(field.writeStatement("value", field.castExpression(fieldValueName))) + .append("\n"); + } } builder.append(" break;\n"); } @@ -350,6 +373,170 @@ private void writeReadBeanGroup( builder.append(" }\n\n"); } + private void writeConstructorRead() { + builder + .append(" private ") + .append(struct.typeName) + .append(" readSchemaConsistentConstructor(ReadContext readContext) {\n"); + builder.append(" Object[] fieldValues = new Object[DESCRIPTORS.size()];\n"); + builder.append(" long[] bufferedFields = newFieldBits(DESCRIPTORS.size());\n"); + builder.append(" beginConstructorRef(readContext);\n"); + builder.append(" try {\n"); + builder.append(" int remaining = countConstructorFields(constructorFieldBits);\n"); + builder.append(" ").append(struct.typeName).append(" value = null;\n"); + builder.append(" if (remaining == 0) {\n"); + builder.append(" value = newConstructorObject(fieldValues);\n"); + builder.append(" referenceConstructorRef(readContext, value);\n"); + builder.append(" }\n"); + builder.append(" for (int i = 0; i < allFields.length; i++) {\n"); + builder.append(" SerializationFieldInfo fieldInfo = allFields[i];\n"); + builder.append(" int fieldId = allFieldIds[i];\n"); + builder.append(" if (hasField(constructorFieldBits, fieldId)) {\n"); + builder.append( + " fieldValues[fieldId] = ctorFieldValue(readContext, readFieldValue(readContext, fieldInfo), type);\n"); + builder.append(" remaining--;\n"); + builder.append(" if (remaining == 0) {\n"); + builder.append(" checkNoUnresolvedReadRef(readContext);\n"); + builder.append(" value = newConstructorObject(fieldValues);\n"); + builder.append(" referenceConstructorRef(readContext, value);\n"); + builder.append(" setBufferedFields(value, fieldValues, bufferedFields);\n"); + builder.append(" }\n"); + builder.append(" } else if (value == null) {\n"); + builder.append( + " fieldValues[fieldId] = bufferFieldValue(readContext, readFieldValue(readContext, fieldInfo), type);\n"); + builder.append(" markField(bufferedFields, fieldId);\n"); + builder.append(" } else {\n"); + builder.append(" readAndSetField(readContext, value, fieldInfo, fieldId);\n"); + builder.append(" }\n"); + builder.append(" }\n"); + builder.append(" if (value == null) {\n"); + builder.append(" checkNoUnresolvedReadRef(readContext);\n"); + builder.append(" value = newConstructorObject(fieldValues);\n"); + builder.append(" referenceConstructorRef(readContext, value);\n"); + builder.append(" setBufferedFields(value, fieldValues, bufferedFields);\n"); + builder.append(" }\n"); + builder.append(" return value;\n"); + builder.append(" } finally {\n"); + builder.append(" endConstructorRef(readContext);\n"); + builder.append(" }\n"); + builder.append(" }\n\n"); + + builder + .append(" private ") + .append(struct.typeName) + .append(" readCompatibleConstructor(ReadContext readContext) {\n"); + builder.append(" Object[] fieldValues = new Object[DESCRIPTORS.size()];\n"); + builder.append(" long[] bufferedFields = newFieldBits(DESCRIPTORS.size());\n"); + builder.append(" beginConstructorRef(readContext);\n"); + builder.append(" try {\n"); + builder.append(" int remaining = countConstructorFields(constructorFieldBits);\n"); + builder.append(" ").append(struct.typeName).append(" value = null;\n"); + builder.append(" if (remaining == 0) {\n"); + builder.append(" value = newConstructorObject(fieldValues);\n"); + builder.append(" referenceConstructorRef(readContext, value);\n"); + builder.append(" }\n"); + builder.append(" for (int i = 0; i < remoteFields.size(); i++) {\n"); + builder.append(" RemoteFieldInfo remoteField = remoteFields.get(i);\n"); + builder.append(" int fieldId = remoteField.matchedId;\n"); + builder.append(" if (fieldId < 0) {\n"); + builder.append(" skipField(readContext, remoteField);\n"); + builder.append(" continue;\n"); + builder.append(" }\n"); + builder.append(" SerializationFieldInfo localField = fieldsById[fieldId];\n"); + builder.append(" if (!canReadRemoteField(remoteField, localField)) {\n"); + builder.append(" skipField(readContext, remoteField);\n"); + builder.append(" continue;\n"); + builder.append(" }\n"); + builder.append( + " Object fieldValue = readCompatibleFieldValue(readContext, remoteField, localField);\n"); + builder.append(" if (hasField(constructorFieldBits, fieldId)) {\n"); + builder.append( + " fieldValues[fieldId] = ctorFieldValue(readContext, fieldValue, type);\n"); + builder.append(" remaining--;\n"); + builder.append(" if (remaining == 0) {\n"); + builder.append(" checkNoUnresolvedReadRef(readContext);\n"); + builder.append(" value = newConstructorObject(fieldValues);\n"); + builder.append(" referenceConstructorRef(readContext, value);\n"); + builder.append(" setBufferedFields(value, fieldValues, bufferedFields);\n"); + builder.append(" }\n"); + builder.append(" } else if (value == null) {\n"); + builder.append( + " fieldValues[fieldId] = bufferFieldValue(readContext, fieldValue, type);\n"); + builder.append(" markField(bufferedFields, fieldId);\n"); + builder.append(" } else {\n"); + builder.append(" setFieldById(value, localField, fieldId, fieldValue);\n"); + builder.append(" }\n"); + builder.append(" }\n"); + builder.append(" if (value == null) {\n"); + builder.append(" checkNoUnresolvedReadRef(readContext);\n"); + builder.append(" value = newConstructorObject(fieldValues);\n"); + builder.append(" referenceConstructorRef(readContext, value);\n"); + builder.append(" setBufferedFields(value, fieldValues, bufferedFields);\n"); + builder.append(" }\n"); + builder.append(" return value;\n"); + builder.append(" } finally {\n"); + builder.append(" endConstructorRef(readContext);\n"); + builder.append(" }\n"); + builder.append(" }\n\n"); + + builder + .append(" private ") + .append(struct.typeName) + .append(" newConstructorObject(Object[] fieldValues) {\n"); + builder + .append(" return (") + .append(struct.typeName) + .append( + ") objectCreator.newInstanceWithArguments(constructorArgs(fieldValues, constructorFieldIds, objectCreator.getConstructorFieldTypes()));\n"); + builder.append(" }\n\n"); + + builder + .append(" private void readAndSetField(ReadContext readContext, ") + .append(struct.typeName) + .append(" value, SerializationFieldInfo fieldInfo, int fieldId) {\n"); + builder.append(" Object fieldValue = readFieldValue(readContext, fieldInfo);\n"); + builder.append(" setFieldById(value, fieldInfo, fieldId, fieldValue);\n"); + builder.append(" }\n\n"); + + builder + .append(" private void setBufferedFields(") + .append(struct.typeName) + .append(" value, Object[] fieldValues, long[] bufferedFields) {\n"); + builder.append(" for (int fieldId = 0; fieldId < fieldsById.length; fieldId++) {\n"); + builder.append(" if (hasField(bufferedFields, fieldId)) {\n"); + builder.append( + " setFieldById(value, fieldsById[fieldId], fieldId, resolveBufferedValue(fieldValues[fieldId], value));\n"); + builder.append(" }\n"); + builder.append(" }\n"); + builder.append(" }\n\n"); + + builder + .append(" private void setFieldById(") + .append(struct.typeName) + .append(" value, SerializationFieldInfo fieldInfo, int fieldId, Object fieldValue) {\n"); + builder.append(" switch (fieldId) {\n"); + for (SourceField field : struct.fields) { + builder.append(" case ").append(field.id).append(":\n"); + builder.append(" "); + if (field.finalField) { + builder + .append("setGeneratedFieldValue(value, fieldInfo, ") + .append(field.castExpression("fieldValue")) + .append(");\n"); + } else { + builder + .append(field.writeStatement("value", field.castExpression("fieldValue"))) + .append("\n"); + } + builder.append(" return;\n"); + } + builder.append(" default:\n"); + builder.append( + " throw new IllegalStateException(\"Unknown generated field id \" + fieldId);\n"); + builder.append(" }\n"); + builder.append(" }\n\n"); + } + private boolean hasDirectWriteField() { for (SourceField field : struct.fields) { if (canEmitDirectWriteField(field)) { @@ -468,6 +655,75 @@ private void appendDirectRead(SourceField field) { builder.append(" ").append(field.writeStatement("value", exactRead)).append("\n"); } + private void appendFinalDirectRead(SourceField field, String targetName, String fieldInfoName) { + if (canEmitDirectStringField(field)) { + builder + .append(" setGeneratedFieldValue(") + .append(targetName) + .append(", ") + .append(fieldInfoName) + .append(", readContext.readString());\n"); + return; + } + if (canEmitDirectArrayField(field)) { + builder.append(" readContext.preserveRefId(-1);\n"); + builder + .append(" setGeneratedFieldValue(") + .append(targetName) + .append(", ") + .append(fieldInfoName) + .append(", ") + .append(field.castExpression("readContext.readNonRef(fieldInfo.typeInfo)")) + .append(");\n"); + return; + } + String exactRead = exactPrimitiveReadExpression(field); + if (exactRead == null) { + appendFinalPrimitiveReadSwitch(field, targetName, fieldInfoName); + return; + } + builder + .append(" setGeneratedFieldValue(") + .append(targetName) + .append(", ") + .append(fieldInfoName) + .append(", ") + .append(exactRead) + .append(");\n"); + } + + private void appendFinalPrimitiveReadSwitch( + SourceField field, String targetName, String fieldInfoName) { + builder.append(" switch (fieldInfo.dispatchId) {\n"); + String[][] cases = primitiveReadCases(field); + for (String[] readCase : cases) { + builder.append(" case DispatchId.").append(readCase[0]).append(":\n"); + builder + .append(" setGeneratedFieldValue(") + .append(targetName) + .append(", ") + .append(fieldInfoName) + .append(", ") + .append(readCase[1]) + .append(");\n"); + builder.append(" break;\n"); + } + builder.append(" default:\n"); + builder + .append(" Object fieldValue") + .append(field.id) + .append(" = readBuildInFieldValue(readContext, fieldInfo);\n"); + builder + .append(" setGeneratedFieldValue(") + .append(targetName) + .append(", ") + .append(fieldInfoName) + .append(", ") + .append(field.castExpression("fieldValue" + field.id)) + .append(");\n"); + builder.append(" }\n"); + } + private void appendPrimitiveReadSwitch(SourceField field) { builder.append(" switch (fieldInfo.dispatchId) {\n"); String[][] cases = primitiveReadCases(field); @@ -759,6 +1015,9 @@ private void writeCompatibleRead() { appendRecordConstructorArguments("field"); builder.append(");\n"); } else { + builder.append(" if (constructorFieldIds != null) {\n"); + builder.append(" return readCompatibleConstructor(readContext);\n"); + builder.append(" }\n"); builder.append(" ").append(struct.typeName).append(" value = newBean();\n"); builder.append(" readContext.reference(value);\n"); builder.append(" for (int i = 0; i < remoteFields.size(); i++) {\n"); @@ -833,10 +1092,19 @@ private void writeCompatibleBeanDispatchGroup(int group) { .append(field.id) .append("]);\n"); appendDebugRemoteRead("after read", "remoteField", 10); - builder - .append(" ") - .append(field.writeStatement("value", field.castExpression("fieldValue"))) - .append("\n"); + builder.append(" "); + if (field.finalField) { + builder + .append("setGeneratedFieldValue(value, fieldsById[") + .append(field.id) + .append("], ") + .append(field.castExpression("fieldValue")) + .append(");\n"); + } else { + builder + .append(field.writeStatement("value", field.castExpression("fieldValue"))) + .append("\n"); + } builder.append(" } else {\n"); appendDebugRemoteRead("before skip", "remoteField", 10); builder.append(" skipField(readContext, remoteField);\n"); @@ -965,6 +1233,11 @@ private void writeCopy() { builder.append(" if (immutable) {\n"); builder.append(" return value;\n"); builder.append(" }\n"); + if (!struct.record) { + builder.append(" if (constructorFieldIds != null) {\n"); + builder.append(" return copyConstructorObject(copyContext, value);\n"); + builder.append(" }\n"); + } if (struct.record) { for (SourceField field : struct.fields) { builder @@ -996,22 +1269,82 @@ private void writeCopy() { builder.append(" ").append(struct.typeName).append(" copied = newBean();\n"); builder.append(" copyContext.reference(value, copied);\n"); for (SourceField field : struct.fields) { + String copiedExpression = + field.castExpression( + "copyFieldValue(copyContext, " + + field.readExpression("value") + + ", fieldsById[" + + field.id + + "])"); builder .append(" ") .append( - field.writeStatement( - "copied", - field.castExpression( - "copyFieldValue(copyContext, " - + field.readExpression("value") - + ", fieldsById[" - + field.id - + "])"))) + field.finalField + ? "setGeneratedFieldValue(copied, fieldsById[" + + field.id + + "], " + + copiedExpression + + ");" + : field.writeStatement("copied", copiedExpression)) .append("\n"); } builder.append(" return copied;\n"); } builder.append(" }\n\n"); + if (!struct.record) { + writeConstructorCopy(); + } + } + + private void writeConstructorCopy() { + builder + .append(" private ") + .append(struct.typeName) + .append(" copyConstructorObject(CopyContext copyContext, ") + .append(struct.typeName) + .append(" value) {\n"); + builder.append(" Object[] fieldValues = new Object[DESCRIPTORS.size()];\n"); + for (SourceField field : struct.fields) { + builder.append(" if (hasField(constructorFieldBits, ").append(field.id).append(")) {\n"); + builder + .append(" fieldValues[") + .append(field.id) + .append("] = copyConstructorFieldValue(copyContext, value, ") + .append(field.readExpression("value")) + .append(", fieldsById[") + .append(field.id) + .append("]);\n"); + builder.append(" }\n"); + } + builder + .append(" ") + .append(struct.typeName) + .append(" copied = newConstructorObject(fieldValues);\n"); + builder.append(" copyContext.reference(value, copied);\n"); + for (SourceField field : struct.fields) { + builder.append(" if (!hasField(constructorFieldBits, ").append(field.id).append(")) {\n"); + String copiedExpression = + field.castExpression( + "copyFieldValue(copyContext, " + + field.readExpression("value") + + ", fieldsById[" + + field.id + + "])"); + builder.append(" "); + if (field.finalField) { + builder + .append("setGeneratedFieldValue(copied, fieldsById[") + .append(field.id) + .append("], ") + .append(copiedExpression) + .append(");\n"); + } else { + builder.append(field.writeStatement("copied", copiedExpression)).append("\n"); + } + builder.append(" }\n"); + } + builder.append(" return copied;\n"); + builder.append(" }\n\n"); } private void writeDescriptorHelpers() { diff --git a/java/fory-annotation-processor/src/test/java/org/apache/fory/annotation/processing/ForyStructProcessorTest.java b/java/fory-annotation-processor/src/test/java/org/apache/fory/annotation/processing/ForyStructProcessorTest.java index 2e4092e14e..a5c5095fad 100644 --- a/java/fory-annotation-processor/src/test/java/org/apache/fory/annotation/processing/ForyStructProcessorTest.java +++ b/java/fory-annotation-processor/src/test/java/org/apache/fory/annotation/processing/ForyStructProcessorTest.java @@ -20,6 +20,7 @@ package org.apache.fory.annotation.processing; import java.io.IOException; +import java.lang.reflect.Constructor; import java.lang.reflect.Field; import java.net.URL; import java.net.URLClassLoader; @@ -115,6 +116,106 @@ public void testLegacyBooleanEvolvingAnnotationCompiles() throws Exception { } } + @Test + public void testStaticAnnotatedConstructor() throws Exception { + CompilationResult result = + compile( + "test.AnnotatedConstructorStruct", + "package test;\n" + + "import org.apache.fory.annotation.ForyConstructor;\n" + + "import org.apache.fory.annotation.ForyStruct;\n" + + "@ForyStruct public class AnnotatedConstructorStruct {\n" + + " private final String name;\n" + + " private final int age;\n" + + " public String note;\n" + + " @ForyConstructor({\"name\", \"age\"})\n" + + " public AnnotatedConstructorStruct(String name, int age) {\n" + + " this.name = name;\n" + + " this.age = age;\n" + + " }\n" + + " public String getName() { return name; }\n" + + " public int getAge() { return age; }\n" + + "}\n"); + Assert.assertTrue(result.success, result.diagnostics()); + String generatedSource = + result.generatedSource("test/AnnotatedConstructorStruct_ForyNativeSerializer.java"); + Assert.assertFalse(generatedSource.contains("boolean[]"), generatedSource); + Assert.assertTrue(generatedSource.contains("long[] constructorFieldBits"), generatedSource); + Assert.assertTrue( + generatedSource.contains("newFieldBits(DESCRIPTORS.size())"), generatedSource); + try (URLClassLoader loader = result.classLoader()) { + Class type = loader.loadClass("test.AnnotatedConstructorStruct"); + Object value = type.getConstructor(String.class, int.class).newInstance("fory", 12); + setField(type, value, "note", "static"); + Fory fory = + Fory.builder() + .withXlang(false) + .withClassLoader(loader) + .withCodegen(false) + .requireClassRegistration(false) + .build(); + Object serializer = fory.getTypeResolver().getTypeInfo(type).getSerializer(); + Assert.assertTrue(serializer instanceof StaticGeneratedStructSerializer); + + Object roundTrip = fory.deserialize(fory.serialize(value)); + Assert.assertEquals(invoke(type, roundTrip, "getName"), "fory"); + Assert.assertEquals(invoke(type, roundTrip, "getAge"), 12); + Assert.assertEquals(getField(type, roundTrip, "note"), "static"); + Object copied = fory.copy(value); + Assert.assertEquals(invoke(type, copied, "getName"), "fory"); + Assert.assertEquals(invoke(type, copied, "getAge"), 12); + Assert.assertEquals(getField(type, copied, "note"), "static"); + } + } + + @Test + @SuppressWarnings({"unchecked", "rawtypes"}) + public void testStaticRegisteredConstructor() throws Exception { + CompilationResult result = + compile( + "test.RegisteredConstructorStruct", + "package test;\n" + + "import org.apache.fory.annotation.ForyStruct;\n" + + "@ForyStruct public class RegisteredConstructorStruct {\n" + + " public static int constructorCalls;\n" + + " private final String name;\n" + + " private final int age;\n" + + " public String note;\n" + + " public RegisteredConstructorStruct(String name, int age) {\n" + + " constructorCalls++;\n" + + " this.name = name;\n" + + " this.age = age;\n" + + " }\n" + + " public String getName() { return name; }\n" + + " public int getAge() { return age; }\n" + + "}\n"); + Assert.assertTrue(result.success, result.diagnostics()); + try (URLClassLoader loader = result.classLoader()) { + Class type = loader.loadClass("test.RegisteredConstructorStruct"); + Constructor constructor = type.getConstructor(String.class, int.class); + Object value = constructor.newInstance("registered", 34); + setField(type, value, "note", "ctor"); + + Fory fory = + Fory.builder() + .withXlang(false) + .withClassLoader(loader) + .withCodegen(false) + .requireClassRegistration(false) + .build(); + fory.registerConstructor(type, constructor, "name", "age"); + Object serializer = fory.getTypeResolver().getTypeInfo(type).getSerializer(); + Assert.assertTrue(serializer instanceof StaticGeneratedStructSerializer); + + setField(type, null, "constructorCalls", 0); + Object roundTrip = fory.deserialize(fory.serialize(value)); + Assert.assertEquals(getField(type, null, "constructorCalls"), 1); + Assert.assertEquals(invoke(type, roundTrip, "getName"), "registered"); + Assert.assertEquals(invoke(type, roundTrip, "getAge"), 34); + Assert.assertEquals(getField(type, roundTrip, "note"), "ctor"); + } + } + @Test public void testForyDebugAnnotationEmitsGeneratedFieldTracing() throws Exception { CompilationResult result = diff --git a/java/fory-core/pom.xml b/java/fory-core/pom.xml index 549dc542dd..5783349a10 100644 --- a/java/fory-core/pom.xml +++ b/java/fory-core/pom.xml @@ -377,32 +377,41 @@ includeantruntime="false" debug="true" release="22"> + + + + + + + + + + + + + + + + + + + + + + + - - - - - prepare-jdk25-test-classes - process-test-classes - - run - - - - - - - - - - - - - - @@ -433,10 +442,13 @@ name="META-INF/versions/25/org/apache/fory/platform/internal/_JDKAccess.class"/> - + + + + + file="${jdk25.mrjar.check.dir}/META-INF/versions/25/org/apache/fory/builder/UnsafeCodegenSupport.class" + property="jdk25.builder.unsafecodegen.present"/> + + + @@ -491,14 +512,23 @@ unless="jdk25.lookup.present" message="JDK25 multi-release _Lookup class is missing from the packaged fory-core jar."/> + unless="jdk25.builder.unsafecodegen.present" + message="JDK25 multi-release UnsafeCodegenSupport class is missing from the packaged fory-core jar."/> + + + @@ -530,34 +560,58 @@ + + + + + + + + + + + + - - org.apache.maven.plugins - maven-surefire-plugin - - ${project.build.directory}/jdk25-test-classes - - diff --git a/java/fory-core/src/main/java/org/apache/fory/AbstractThreadSafeFory.java b/java/fory-core/src/main/java/org/apache/fory/AbstractThreadSafeFory.java index 8070a7d1cf..9d9f427201 100644 --- a/java/fory-core/src/main/java/org/apache/fory/AbstractThreadSafeFory.java +++ b/java/fory-core/src/main/java/org/apache/fory/AbstractThreadSafeFory.java @@ -19,6 +19,7 @@ package org.apache.fory; +import java.lang.reflect.Constructor; import java.util.function.Function; import org.apache.fory.resolver.TypeChecker; import org.apache.fory.resolver.TypeResolver; @@ -66,6 +67,12 @@ public void register(ForyModule module) { registerCallback(fory -> fory.register(module)); } + @Override + public void registerConstructor( + Class type, Constructor constructor, String... fieldNames) { + registerCallback(fory -> fory.registerConstructor(type, constructor, fieldNames)); + } + public void registerUnion( Class cls, int id, org.apache.fory.serializer.Serializer serializer) { registerCallback(fory -> fory.registerUnion(cls, id, serializer)); diff --git a/java/fory-core/src/main/java/org/apache/fory/BaseFory.java b/java/fory-core/src/main/java/org/apache/fory/BaseFory.java index d214ac5334..0dc0cc0623 100644 --- a/java/fory-core/src/main/java/org/apache/fory/BaseFory.java +++ b/java/fory-core/src/main/java/org/apache/fory/BaseFory.java @@ -20,6 +20,7 @@ package org.apache.fory; import java.io.OutputStream; +import java.lang.reflect.Constructor; import java.nio.ByteBuffer; import java.util.function.Function; import org.apache.fory.io.ForyInputStream; @@ -89,6 +90,17 @@ public interface BaseFory { */ void register(ForyModule module); + /** + * Register an explicit constructor-to-field mapping for {@code type}. + * + *

    The constructor arguments are populated from {@code fieldNames} in order. This is useful for + * third-party classes that cannot annotate a constructor with {@code @ForyConstructor}. + * + *

    Call this during setup before top-level serialization, deserialization, or copy operations + * start. + */ + void registerConstructor(Class type, Constructor constructor, String... fieldNames); + void registerUnion(Class cls, int id, Serializer serializer); void registerUnion(Class cls, String namespace, String typeName, Serializer serializer); diff --git a/java/fory-core/src/main/java/org/apache/fory/Fory.java b/java/fory-core/src/main/java/org/apache/fory/Fory.java index d47cbc4e5c..695774ce5e 100644 --- a/java/fory-core/src/main/java/org/apache/fory/Fory.java +++ b/java/fory-core/src/main/java/org/apache/fory/Fory.java @@ -21,6 +21,7 @@ import java.io.IOException; import java.io.OutputStream; +import java.lang.reflect.Constructor; import java.nio.ByteBuffer; import java.util.IdentityHashMap; import java.util.function.Consumer; @@ -211,6 +212,13 @@ public void register(ForyModule module) { } } + @Override + public void registerConstructor( + Class type, Constructor constructor, String... fieldNames) { + checkRegisterAllowed(); + sharedRegistry.getObjectCreatorRegistry().registerConstructor(type, constructor, fieldNames); + } + @Override public void registerUnion(Class cls, int id, Serializer serializer) { getTypeResolver().registerUnion(cls, Integer.toUnsignedLong(id), serializer); @@ -659,6 +667,15 @@ SharedRegistry getSharedRegistry() { return sharedRegistry; } + private void checkRegisterAllowed() { + if (typeResolver.isRegistrationFinished()) { + throw new ForyException( + "Cannot register class/serializer after registration has been frozen. Please register " + + "all classes before invoking top-level `serialize/deserialize/copy` methods of " + + "Fory."); + } + } + public Config getConfig() { return config; } diff --git a/java/fory-core/src/main/java/org/apache/fory/annotation/ForyConstructor.java b/java/fory-core/src/main/java/org/apache/fory/annotation/ForyConstructor.java new file mode 100644 index 0000000000..8ce7dd5ef2 --- /dev/null +++ b/java/fory-core/src/main/java/org/apache/fory/annotation/ForyConstructor.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.annotation; + +import java.lang.annotation.Documented; +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** Maps one constructor's arguments to serialized field names. */ +@Documented +@Retention(RetentionPolicy.RUNTIME) +@Target(ElementType.CONSTRUCTOR) +@Public +public @interface ForyConstructor { + /** + * Field names in constructor argument order. + * + *

    Every name must refer to one non-static serialized field declared by the target class or a + * superclass. Duplicate field names in a class hierarchy are not bindable by this annotation. + */ + String[] value(); +} diff --git a/java/fory-core/src/main/java/org/apache/fory/builder/AccessorHelper.java b/java/fory-core/src/main/java/org/apache/fory/builder/AccessorHelper.java index aa76fa7c5b..f2979d5361 100644 --- a/java/fory-core/src/main/java/org/apache/fory/builder/AccessorHelper.java +++ b/java/fory-core/src/main/java/org/apache/fory/builder/AccessorHelper.java @@ -43,7 +43,7 @@ /** * Define accessor helper methods in beanClass's classloader and same package to avoid reflective - * call overhead. {@link sun.misc.Unsafe} is another method to avoid reflection cost. + * call overhead. */ public class AccessorHelper { private static final Logger LOG = LoggerFactory.getLogger(AccessorHelper.class); diff --git a/java/fory-core/src/main/java/org/apache/fory/builder/BaseObjectCodecBuilder.java b/java/fory-core/src/main/java/org/apache/fory/builder/BaseObjectCodecBuilder.java index bdd81edaf9..78cc01953d 100644 --- a/java/fory-core/src/main/java/org/apache/fory/builder/BaseObjectCodecBuilder.java +++ b/java/fory-core/src/main/java/org/apache/fory/builder/BaseObjectCodecBuilder.java @@ -121,7 +121,8 @@ import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.meta.TypeExtMeta; import org.apache.fory.platform.GraalvmSupport; -import org.apache.fory.platform.internal._JDKAccess; +import org.apache.fory.reflect.ObjectCreator; +import org.apache.fory.reflect.ObjectCreators; import org.apache.fory.reflect.ReflectionUtils; import org.apache.fory.reflect.TypeRef; import org.apache.fory.resolver.ClassResolver; @@ -153,7 +154,6 @@ import org.apache.fory.type.Types; import org.apache.fory.util.Preconditions; import org.apache.fory.util.StringUtils; -import sun.misc.Unsafe; /** * Generate sequential read/write code for java serialization to speed up performance. It also @@ -276,6 +276,27 @@ protected static T typeResolver(Fory fory, Function functio return fory.getJITContext().asyncVisitFory(f -> function.apply(f.getTypeResolver())); } + @Override + protected void cacheObjectCreator(Class type) { + typeResolver.getObjectCreator(type); + } + + @Override + protected Expression getObjectCreator(Class type) { + cacheObjectCreator(type); + return getOrCreateField( + false, + ObjectCreator.class, + ctx.newName("objectCreator_" + type.getSimpleName()), + () -> + new StaticInvoke( + ObjectCreators.class, + "getObjectCreator", + TypeRef.of(ObjectCreator.class), + typeResolverRef, + getClassExpr(type))); + } + protected boolean needWriteRef(TypeRef type) { return typeResolver(r -> r.needToWriteRef(type)); } @@ -420,13 +441,7 @@ protected void registerJITNotifyCallback() { * @see CodeGenerator#getClassUniqueId */ protected void addCommonImports() { - ctx.addImports( - Fory.class, - MemoryBuffer.class, - WriteContext.class, - ReadContext.class, - _JDKAccess.class, - Unsafe.class); + ctx.addImports(Fory.class, MemoryBuffer.class, WriteContext.class, ReadContext.class); ctx.addImports(TypeInfo.class, TypeInfoHolder.class, ClassResolver.class); ctx.addImport(Generated.class); ctx.addImports(LazyInitBeanSerializer.class, EnumSerializer.class); diff --git a/java/fory-core/src/main/java/org/apache/fory/builder/CodecBuilder.java b/java/fory-core/src/main/java/org/apache/fory/builder/CodecBuilder.java index dca762854b..c52d771e80 100644 --- a/java/fory-core/src/main/java/org/apache/fory/builder/CodecBuilder.java +++ b/java/fory-core/src/main/java/org/apache/fory/builder/CodecBuilder.java @@ -43,6 +43,7 @@ import java.util.Set; import java.util.function.Supplier; import java.util.stream.Collectors; +import org.apache.fory.codegen.Code; import org.apache.fory.codegen.CodegenContext; import org.apache.fory.codegen.Expression; import org.apache.fory.codegen.Expression.Cast; @@ -57,7 +58,6 @@ import org.apache.fory.memory.NativeByteOrder; import org.apache.fory.platform.GraalvmSupport; import org.apache.fory.platform.JdkVersion; -import org.apache.fory.platform.internal._JDKAccess; import org.apache.fory.reflect.FieldAccessor; import org.apache.fory.reflect.ObjectCreator; import org.apache.fory.reflect.ObjectCreators; @@ -71,7 +71,6 @@ import org.apache.fory.util.function.Functions; import org.apache.fory.util.record.RecordComponent; import org.apache.fory.util.record.RecordUtils; -import sun.misc.Unsafe; /** * Base builder for generating code to serialize java bean in row-format or object stream format. @@ -333,12 +332,11 @@ private Expression unsafeAccessField( Preconditions.checkArgument(!fieldNullable); TypeRef returnType = descriptor.getTypeRef(); String funcName = "get" + StringUtils.capitalize(descriptor.getRawType().toString()); - return new Invoke(getUnsafe(), funcName, returnType, false, inputObject, fieldOffsetExpr); + return unsafeInvoke(funcName, returnType, false, inputObject, fieldOffsetExpr); } else { // ex: Unsafe.getObject(obj, fieldOffset) Invoke getObj = - new Invoke( - getUnsafe(), "getObject", OBJECT_TYPE, fieldNullable, inputObject, fieldOffsetExpr); + unsafeInvoke("getObject", OBJECT_TYPE, fieldNullable, inputObject, fieldOffsetExpr); return tryCastIfPublic(getObj, descriptor.getTypeRef(), fieldName); } } @@ -356,21 +354,66 @@ private Expression fieldOffsetExpr(Class cls, Descriptor descriptor) { Expression classExpr = beanClassExpr(field.getDeclaringClass()); new Invoke(classExpr, "getDeclaredField", TypeRef.of(Field.class)); Expression reflectFieldRef = getReflectField(field.getDeclaringClass(), field, false); - return new Invoke( - getUnsafe(), "objectFieldOffset", PRIMITIVE_LONG_TYPE, reflectFieldRef) - .inline(); + return unsafeInvoke("objectFieldOffset", PRIMITIVE_LONG_TYPE, reflectFieldRef).inline(); }); } else { - return Literal.ofLong(_JDKAccess.UNSAFE.objectFieldOffset(field)); + return Literal.ofLong(UnsafeCodegenSupport.objectFieldOffset(field)); } } private Reference getUnsafe() { - return getOrCreateField( - true, - Unsafe.class, - "_unsafe_", - () -> new StaticInvoke(_JDKAccess.class, "unsafe", TypeRef.of(Unsafe.class))); + String fieldName = "_unsafe_"; + Reference fieldRef = fieldMap.get(fieldName); + if (fieldRef == null) { + String uniqueFieldName = ctx.newName(fieldName); + ctx.addField( + true, + true, + UnsafeCodegenSupport.unsafeTypeName(), + uniqueFieldName, + new UnsafeInitExpression()); + fieldRef = new Reference(uniqueFieldName, OBJECT_TYPE); + fieldMap.put(fieldName, fieldRef); + } + return fieldRef; + } + + private Invoke unsafeInvoke(String functionName, TypeRef returnType, Expression... arguments) { + return unsafeInvoke(functionName, returnType, false, arguments); + } + + private Invoke unsafeInvoke( + String functionName, TypeRef returnType, boolean returnNullable, Expression... arguments) { + return new Invoke( + getUnsafe(), + functionName, + "", + returnType, + returnNullable, + "allocateInstance".equals(functionName), + arguments); + } + + private Invoke unsafeInvoke(String functionName, Expression... arguments) { + return new Invoke(getUnsafe(), functionName, "", PRIMITIVE_VOID_TYPE, false, false, arguments); + } + + private static final class UnsafeInitExpression extends Expression.AbstractExpression { + private UnsafeInitExpression() { + super(new Expression[0]); + } + + @Override + public TypeRef type() { + return OBJECT_TYPE; + } + + @Override + public Code.ExprCode doGenCode(CodegenContext ctx) { + return new Code.ExprCode( + Code.LiteralValue.FalseLiteral, + Code.exprValue(Object.class, UnsafeCodegenSupport.unsafeInitCode())); + } } private Reference getFieldAccessor(Descriptor descriptor) { @@ -477,9 +520,9 @@ private Expression unsafeSetField(Expression bean, Descriptor descriptor, Expres if (descriptor.getTypeRef().isPrimitive()) { Preconditions.checkArgument(getRawType(value.type()) == getRawType(fieldType)); String funcName = "put" + StringUtils.capitalize(getRawType(fieldType).toString()); - return new Invoke(getUnsafe(), funcName, bean, fieldOffsetExpr, value); + return unsafeInvoke(funcName, bean, fieldOffsetExpr, value); } else { - return new Invoke(getUnsafe(), "putObject", bean, fieldOffsetExpr, value); + return unsafeInvoke("putObject", bean, fieldOffsetExpr, value); } } @@ -549,18 +592,21 @@ protected Expression newBean() { return new Expression.NewInstance(beanType); } else { if (JdkVersion.MAJOR_VERSION >= 25) { - ObjectCreators.getObjectCreator(beanClass); // trigger cache + cacheObjectCreator(beanClass); // trigger cache Invoke newInstance = new Invoke(getObjectCreator(beanClass), "newInstance", OBJECT_TYPE); return sourcePublicAccessible(beanClass) ? new Cast(newInstance, beanType) : newInstance; } - Invoke newInstance = - new Invoke(getUnsafe(), "allocateInstance", OBJECT_TYPE, beanClassExpr()); + Invoke newInstance = unsafeInvoke("allocateInstance", OBJECT_TYPE, beanClassExpr()); return sourcePublicAccessible(beanClass) ? new Cast(newInstance, beanType) : newInstance; } } + protected void cacheObjectCreator(Class type) { + ObjectCreators.getObjectCreator(type); + } + protected Expression getObjectCreator(Class type) { - ObjectCreators.getObjectCreator(type); // trigger cache + cacheObjectCreator(type); return getOrCreateField( true, ObjectCreator.class, @@ -654,49 +700,49 @@ private StaticInvoke inlineReflectionUtilsInvoke( /** Build unsafePut operation. */ protected Expression unsafePut(Expression base, Expression pos, Expression value) { - return new Invoke(getUnsafe(), "putByte", base, pos, value); + return unsafeInvoke("putByte", base, pos, value); } protected Expression unsafePutBoolean(Expression base, Expression pos, Expression value) { - return new Invoke(getUnsafe(), "putBoolean", base, pos, value); + return unsafeInvoke("putBoolean", base, pos, value); } protected Expression unsafePutChar(Expression base, Expression pos, Expression value) { - return new Invoke(getUnsafe(), "putChar", base, pos, value); + return unsafeInvoke("putChar", base, pos, value); } protected Expression unsafePutShort(Expression base, Expression pos, Expression value) { - return new Invoke(getUnsafe(), "putShort", base, pos, value); + return unsafeInvoke("putShort", base, pos, value); } protected Expression unsafePutInt(Expression base, Expression pos, Expression value) { - return new Invoke(getUnsafe(), "putInt", base, pos, value); + return unsafeInvoke("putInt", base, pos, value); } protected Expression unsafePutLong(Expression base, Expression pos, Expression value) { - return new Invoke(getUnsafe(), "putLong", base, pos, value); + return unsafeInvoke("putLong", base, pos, value); } protected Expression unsafePutFloat(Expression base, Expression pos, Expression value) { - return new Invoke(getUnsafe(), "putFloat", base, pos, value); + return unsafeInvoke("putFloat", base, pos, value); } /** Build unsafePutDouble operation. */ protected Expression unsafePutDouble(Expression base, Expression pos, Expression value) { - return new Invoke(getUnsafe(), "putDouble", base, pos, value); + return unsafeInvoke("putDouble", base, pos, value); } /** Build unsafeGet operation. */ protected Expression unsafeGet(Expression base, Expression pos) { - return new Invoke(getUnsafe(), "getByte", PRIMITIVE_BYTE_TYPE, base, pos); + return unsafeInvoke("getByte", PRIMITIVE_BYTE_TYPE, base, pos); } protected Expression unsafeGetBoolean(Expression base, Expression pos) { - return new Invoke(getUnsafe(), "getBoolean", PRIMITIVE_BOOLEAN_TYPE, base, pos); + return unsafeInvoke("getBoolean", PRIMITIVE_BOOLEAN_TYPE, base, pos); } protected Expression unsafeGetChar(Expression base, Expression pos) { - Inlineable expr = new Invoke(getUnsafe(), "getChar", PRIMITIVE_CHAR_TYPE, base, pos); + Inlineable expr = unsafeInvoke("getChar", PRIMITIVE_CHAR_TYPE, base, pos); if (!NativeByteOrder.IS_LITTLE_ENDIAN) { expr = new StaticInvoke(Character.class, "reverseBytes", PRIMITIVE_CHAR_TYPE, expr.inline()); } @@ -704,7 +750,7 @@ protected Expression unsafeGetChar(Expression base, Expression pos) { } protected Expression unsafeGetShort(Expression base, Expression pos) { - Inlineable expr = new Invoke(getUnsafe(), "getShort", PRIMITIVE_SHORT_TYPE, base, pos); + Inlineable expr = unsafeInvoke("getShort", PRIMITIVE_SHORT_TYPE, base, pos); if (!NativeByteOrder.IS_LITTLE_ENDIAN) { expr = new StaticInvoke(Short.class, "reverseBytes", PRIMITIVE_SHORT_TYPE, expr.inline()); } @@ -712,7 +758,7 @@ protected Expression unsafeGetShort(Expression base, Expression pos) { } protected Expression unsafeGetInt(Expression base, Expression pos) { - Inlineable expr = new Invoke(getUnsafe(), "getInt", PRIMITIVE_INT_TYPE, base, pos); + Inlineable expr = unsafeInvoke("getInt", PRIMITIVE_INT_TYPE, base, pos); if (!NativeByteOrder.IS_LITTLE_ENDIAN) { expr = new StaticInvoke(Integer.class, "reverseBytes", PRIMITIVE_INT_TYPE, expr.inline()); } @@ -720,7 +766,7 @@ protected Expression unsafeGetInt(Expression base, Expression pos) { } protected Expression unsafeGetLong(Expression base, Expression pos) { - Inlineable expr = new Invoke(getUnsafe(), "getLong", PRIMITIVE_LONG_TYPE, base, pos); + Inlineable expr = unsafeInvoke("getLong", PRIMITIVE_LONG_TYPE, base, pos); if (!NativeByteOrder.IS_LITTLE_ENDIAN) { expr = new StaticInvoke(Long.class, "reverseBytes", PRIMITIVE_LONG_TYPE, expr.inline()); } @@ -728,7 +774,7 @@ protected Expression unsafeGetLong(Expression base, Expression pos) { } protected Expression unsafeGetFloat(Expression base, Expression pos) { - Inlineable expr = new Invoke(getUnsafe(), "getInt", PRIMITIVE_INT_TYPE, base, pos); + Inlineable expr = unsafeInvoke("getInt", PRIMITIVE_INT_TYPE, base, pos); if (!NativeByteOrder.IS_LITTLE_ENDIAN) { expr = new StaticInvoke(Integer.class, "reverseBytes", PRIMITIVE_INT_TYPE, expr.inline()); } @@ -736,7 +782,7 @@ protected Expression unsafeGetFloat(Expression base, Expression pos) { } protected Expression unsafeGetDouble(Expression base, Expression pos) { - Inlineable expr = new Invoke(getUnsafe(), "getLong", PRIMITIVE_LONG_TYPE, base, pos); + Inlineable expr = unsafeInvoke("getLong", PRIMITIVE_LONG_TYPE, base, pos); if (!NativeByteOrder.IS_LITTLE_ENDIAN) { expr = new StaticInvoke(Long.class, "reverseBytes", PRIMITIVE_LONG_TYPE, expr.inline()); } diff --git a/java/fory-core/src/main/java/org/apache/fory/builder/CompatibleCodecBuilder.java b/java/fory-core/src/main/java/org/apache/fory/builder/CompatibleCodecBuilder.java index ffeaab8100..676b7a999a 100644 --- a/java/fory-core/src/main/java/org/apache/fory/builder/CompatibleCodecBuilder.java +++ b/java/fory-core/src/main/java/org/apache/fory/builder/CompatibleCodecBuilder.java @@ -43,7 +43,6 @@ import org.apache.fory.meta.TypeDef; import org.apache.fory.platform.GraalvmSupport; import org.apache.fory.reflect.ObjectCreator; -import org.apache.fory.reflect.ObjectCreators; import org.apache.fory.reflect.TypeRef; import org.apache.fory.resolver.TypeResolver; import org.apache.fory.serializer.CodegenSerializer; @@ -370,7 +369,7 @@ private boolean constructorOwnsField(Member member) { if (constructorFieldIndexes == null) { return false; } - ObjectCreator objectCreator = ObjectCreators.getObjectCreator(beanClass); + ObjectCreator objectCreator = typeResolver.getObjectCreator(beanClass); String[] names = objectCreator.getConstructorFieldNames(); Class[] declaringClasses = objectCreator.getConstructorFieldDeclaringClasses(); for (int i = 0; i < names.length; i++) { @@ -385,7 +384,7 @@ private boolean constructorOwnsField(Member member) { @Override protected Expression defaultConstructorValue(int constructorParameterIndex) { - ObjectCreator objectCreator = ObjectCreators.getObjectCreator(beanClass); + ObjectCreator objectCreator = typeResolver.getObjectCreator(beanClass); String fieldName = objectCreator.getConstructorFieldNames()[constructorParameterIndex]; Class[] declaringClasses = objectCreator.getConstructorFieldDeclaringClasses(); Class declaringClass = diff --git a/java/fory-core/src/main/java/org/apache/fory/builder/ObjectCodecBuilder.java b/java/fory-core/src/main/java/org/apache/fory/builder/ObjectCodecBuilder.java index b1d25fcbff..04bce6bce3 100644 --- a/java/fory-core/src/main/java/org/apache/fory/builder/ObjectCodecBuilder.java +++ b/java/fory-core/src/main/java/org/apache/fory/builder/ObjectCodecBuilder.java @@ -66,7 +66,6 @@ import org.apache.fory.meta.TypeDef; import org.apache.fory.platform.JdkVersion; import org.apache.fory.reflect.ObjectCreator; -import org.apache.fory.reflect.ObjectCreators; import org.apache.fory.reflect.TypeRef; import org.apache.fory.serializer.AbstractObjectSerializer; import org.apache.fory.serializer.ObjectSerializer; @@ -176,7 +175,7 @@ protected final void initConstructorFields( boolean allowMissingNonFinal, String[] defaultFields, Class[] defaultDeclaringClasses) { - ObjectCreator objectCreator = ObjectCreators.getObjectCreator(beanClass); + ObjectCreator objectCreator = typeResolver.getObjectCreator(beanClass); if (!objectCreator.hasConstructorFields()) { return; } @@ -1035,7 +1034,7 @@ public Expression buildDecodeExpression() { FieldsCollector collector = (FieldsCollector) bean; bean = createRecord(collector.recordValuesMap); } else { - ObjectCreators.getObjectCreator(beanClass); // trigger cache and make error raised early + typeResolver.getObjectCreator(beanClass); // trigger cache and make error raised early bean = new Invoke(getObjectCreator(beanClass), "newInstanceWithArguments", OBJECT_TYPE, bean); } @@ -1064,6 +1063,7 @@ private Expression buildConstructorDecodeExpression( int index = fieldIndexes.get(descriptor); walkPath.add(descriptor.getDeclaringClass() + descriptor.getName()); if (constructorFieldMask[index]) { + trackConstructorRefRead(expressions, buffer, descriptor); expressions.add(deserializeToFieldsArray(fieldsArray, buffer, descriptor, true)); remainingConstructorFields--; if (remainingConstructorFields == 0) { @@ -1071,6 +1071,7 @@ private Expression buildConstructorDecodeExpression( addBufferedFieldSetters(expressions, bean, fieldsArray, bufferedNonConstructorFields); } } else if (bean == null) { + trackConstructorRefRead(expressions, buffer, descriptor); expressions.add(deserializeToFieldsArray(fieldsArray, buffer, descriptor, false)); bufferedNonConstructorFields.add(descriptor); } else { @@ -1106,6 +1107,19 @@ private List protocolDescriptors() { return descriptors; } + private void trackConstructorRefRead( + ListExpression expressions, Reference buffer, Descriptor descriptor) { + if (descriptor.isTrackingRef()) { + expressions.add( + new StaticInvoke( + AbstractObjectSerializer.class, + "trackConstructorRefRead", + PRIMITIVE_VOID_TYPE, + readContextRef(), + buffer)); + } + } + private void addDescriptors(List descriptors, List> groups) { for (List group : groups) { descriptors.addAll(group); @@ -1207,7 +1221,7 @@ protected Expression createConstructorObject(FieldsArray fieldValues) { } directParams[i] = tryInlineCast(params[i], TypeRef.of(constructorFieldTypes[i])); } - ObjectCreator objectCreator = ObjectCreators.getObjectCreator(beanClass); + ObjectCreator objectCreator = typeResolver.getObjectCreator(beanClass); if (JdkVersion.MAJOR_VERSION >= 25 && objectCreator.isOnlyPublicConstructor() && sourcePublicAccessible(beanClass) diff --git a/java/fory-core/src/main/java/org/apache/fory/builder/UnsafeCodegenSupport.java b/java/fory-core/src/main/java/org/apache/fory/builder/UnsafeCodegenSupport.java new file mode 100644 index 0000000000..59390a2f04 --- /dev/null +++ b/java/fory-core/src/main/java/org/apache/fory/builder/UnsafeCodegenSupport.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.builder; + +import java.lang.reflect.Field; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import org.apache.fory.annotation.Internal; +import org.apache.fory.platform.JdkVersion; + +/** Internal bridge for JDK8-24 generated code that still uses Unsafe fast paths. */ +@Internal +public final class UnsafeCodegenSupport { + private static final Object UNSAFE; + private static final Method OBJECT_FIELD_OFFSET; + + static { + if (JdkVersion.MAJOR_VERSION >= 25) { + UNSAFE = null; + OBJECT_FIELD_OFFSET = null; + } else { + try { + Class unsafeClass = Class.forName("sun.misc.Unsafe"); + Field unsafeField = unsafeClass.getDeclaredField("theUnsafe"); + unsafeField.setAccessible(true); + UNSAFE = unsafeField.get(null); + OBJECT_FIELD_OFFSET = unsafeClass.getMethod("objectFieldOffset", Field.class); + } catch (ReflectiveOperationException e) { + throw new UnsupportedOperationException("Unsafe is not supported in this platform.", e); + } + } + } + + private UnsafeCodegenSupport() {} + + public static Object unsafe() { + checkUnsafeSupported(); + return UNSAFE; + } + + public static long objectFieldOffset(Field field) { + checkUnsafeSupported(); + try { + return (long) OBJECT_FIELD_OFFSET.invoke(UNSAFE, field); + } catch (IllegalAccessException e) { + throw new IllegalStateException("Unsafe objectFieldOffset is not accessible.", e); + } catch (InvocationTargetException e) { + throw new RuntimeException(e.getCause()); + } + } + + static String unsafeTypeName() { + checkUnsafeSupported(); + return "sun.misc.Unsafe"; + } + + static String unsafeInitCode() { + checkUnsafeSupported(); + return "(sun.misc.Unsafe) " + UnsafeCodegenSupport.class.getName() + ".unsafe()"; + } + + private static void checkUnsafeSupported() { + if (JdkVersion.MAJOR_VERSION >= 25) { + throw new UnsupportedOperationException("Generated Unsafe access is unsupported on JDK25+"); + } + } +} diff --git a/java/fory-core/src/main/java/org/apache/fory/context/CopyContext.java b/java/fory-core/src/main/java/org/apache/fory/context/CopyContext.java index f17c170ca7..5eea194405 100644 --- a/java/fory-core/src/main/java/org/apache/fory/context/CopyContext.java +++ b/java/fory-core/src/main/java/org/apache/fory/context/CopyContext.java @@ -21,7 +21,6 @@ import java.util.Arrays; import org.apache.fory.collection.IdentityMap; -import org.apache.fory.exception.ForyException; import org.apache.fory.resolver.ClassResolver; import org.apache.fory.resolver.TypeInfo; import org.apache.fory.resolver.TypeResolver; @@ -42,7 +41,6 @@ public final class CopyContext { private final TypeResolver typeResolver; private final boolean copyRefTracking; private final IdentityMap originToCopyMap; - private static final Object COPY_IN_PROGRESS = new Object(); /** * Creates a copy context for one runtime. @@ -90,26 +88,7 @@ public void reference(T origin, T copied) { /** Returns the previously registered copy for {@code origin}, or {@code null} if absent. */ public T getCopyObject(T origin) { - Object copied = originToCopyMap.get(origin); - if (copied == COPY_IN_PROGRESS) { - throw new ForyException( - "Cyclic references to constructor-bound objects cannot be copied before construction."); - } - return (T) copied; - } - - /** Marks {@code origin} as being copied before a constructor-bound copy can be registered. */ - public void markCopying(Object origin) { - if (copyRefTracking && origin != null) { - originToCopyMap.put(origin, COPY_IN_PROGRESS); - } - } - - /** Clears an in-progress constructor-bound copy marker after a failed copy. */ - public void cancelCopy(Object origin) { - if (copyRefTracking && origin != null && originToCopyMap.get(origin) == COPY_IN_PROGRESS) { - originToCopyMap.remove(origin); - } + return (T) originToCopyMap.get(origin); } /** diff --git a/java/fory-core/src/main/java/org/apache/fory/context/MapRefReader.java b/java/fory-core/src/main/java/org/apache/fory/context/MapRefReader.java index 328acbf78d..59101b61ac 100644 --- a/java/fory-core/src/main/java/org/apache/fory/context/MapRefReader.java +++ b/java/fory-core/src/main/java/org/apache/fory/context/MapRefReader.java @@ -38,8 +38,6 @@ public final class MapRefReader implements RefReader { private long readTotalObjectSize = 0; private final ObjectArray readObjects = new ObjectArray(DEFAULT_ARRAY_CAPACITY); private final IntArray readRefIds = new IntArray(DEFAULT_ARRAY_CAPACITY); - private final IntArray trackedRefIds = new IntArray(DEFAULT_ARRAY_CAPACITY); - private final IntArray unresolvedRefIds = new IntArray(DEFAULT_ARRAY_CAPACITY); private Object readObject; /** Reads a ref-or-null header and resolves cached references immediately when present. */ @@ -47,7 +45,7 @@ public final class MapRefReader implements RefReader { public byte readRefOrNull(MemoryBuffer buffer) { byte headFlag = buffer.readByte(); if (headFlag == Fory.REF_FLAG) { - readObject = readRef(buffer.readVarUInt32Small14()); + readObject = getReadRef(buffer.readVarUInt32Small14()); } else { readObject = null; } @@ -75,7 +73,7 @@ public int preserveRefId(int refId) { public int tryPreserveRefId(MemoryBuffer buffer) { byte headFlag = buffer.readByte(); if (headFlag == Fory.REF_FLAG) { - readObject = readRef(buffer.readVarUInt32Small14()); + readObject = getReadRef(buffer.readVarUInt32Small14()); } else { readObject = null; if (headFlag == Fory.REF_VALUE_FLAG) { @@ -106,26 +104,6 @@ public void reference(Object object) { setReadRef(refId, object); } - /** Binds a reserved ref id that may no longer be the top of the pending-id stack. */ - @Override - public void reference(int refId, Object object) { - removePreservedRefId(refId); - setReadRef(refId, object); - } - - private void removePreservedRefId(int refId) { - for (int i = readRefIds.size - 1; i >= 0; i--) { - if (readRefIds.elementData[i] == refId) { - int numMoved = readRefIds.size - i - 1; - if (numMoved > 0) { - System.arraycopy(readRefIds.elementData, i + 1, readRefIds.elementData, i, numMoved); - } - readRefIds.size--; - return; - } - } - } - /** Returns the previously materialized object stored at {@code id}. */ @Override public Object getReadRef(int id) { @@ -138,26 +116,6 @@ public Object getReadRef() { return readObject; } - private Object readRef(int id) { - if (trackedRefIds.size == 0) { - return readObjects.get(id); - } - Object object = readObjects.get(id); - if (object == null && isTrackedRef(id)) { - unresolvedRefIds.add(id); - } - return object; - } - - private boolean isTrackedRef(int id) { - for (int i = trackedRefIds.size - 1; i >= 0; i--) { - if (trackedRefIds.get(i) == id) { - return true; - } - } - return false; - } - /** Stores {@code object} under an already reserved read ref id. */ @Override public void setReadRef(int id, Object object) { @@ -166,44 +124,6 @@ public void setReadRef(int id, Object object) { } } - @Override - public void trackUnresolvedRef(int id) { - trackedRefIds.add(id); - } - - @Override - public boolean hasTrackedRef() { - return trackedRefIds.size > 0; - } - - @Override - public int currentTrackedRefId() { - return trackedRefIds.get(trackedRefIds.size - 1); - } - - @Override - public void untrackUnresolvedRef() { - if (trackedRefIds.size > 0) { - trackedRefIds.pop(); - } - } - - @Override - public boolean consumeUnresolvedRef(int id) { - boolean found = false; - int newSize = 0; - for (int i = 0; i < unresolvedRefIds.size; i++) { - int unresolvedRefId = unresolvedRefIds.get(i); - if (unresolvedRefId == id) { - found = true; - } else { - unresolvedRefIds.elementData[newSize++] = unresolvedRefId; - } - } - unresolvedRefIds.size = newSize; - return found; - } - /** Exposes the resolved read-reference table for debugging and focused tests. */ public ObjectArray getReadRefs() { return readObjects; @@ -226,8 +146,6 @@ public void reset() { } readObjects.clearApproximate(avg); readRefIds.clear(); - trackedRefIds.clear(); - unresolvedRefIds.clear(); readObject = null; } } diff --git a/java/fory-core/src/main/java/org/apache/fory/context/ReadContext.java b/java/fory-core/src/main/java/org/apache/fory/context/ReadContext.java index 3a15bb6e28..6dca2e503a 100644 --- a/java/fory-core/src/main/java/org/apache/fory/context/ReadContext.java +++ b/java/fory-core/src/main/java/org/apache/fory/context/ReadContext.java @@ -360,11 +360,6 @@ public void reference(Object object) { refReader.reference(object); } - /** Binds a specific preserved read ref id to {@code object}. */ - public void reference(int refId, Object object) { - refReader.reference(refId, object); - } - /** Returns a previously read object by ref id. */ public Object getReadRef(int id) { return refReader.getReadRef(id); @@ -380,31 +375,6 @@ public void setReadRef(int id, Object object) { refReader.setReadRef(id, object); } - /** Starts tracking unresolved reads of {@code id} while a constructor-bound object is read. */ - public void trackUnresolvedRef(int id) { - refReader.trackUnresolvedRef(id); - } - - /** Returns whether a constructor-bound object ref id is currently tracked. */ - public boolean hasTrackedRef() { - return refReader.hasTrackedRef(); - } - - /** Returns the active constructor-bound object ref id. */ - public int currentTrackedRefId() { - return refReader.currentTrackedRefId(); - } - - /** Stops tracking unresolved reads for the most recent constructor-bound object. */ - public void untrackUnresolvedRef() { - refReader.untrackUnresolvedRef(); - } - - /** Returns and clears whether {@code id} was read before it was bound to an object. */ - public boolean consumeUnresolvedRef(int id) { - return refReader.consumeUnresolvedRef(id); - } - /** Returns the read-side meta-string state for the current runtime. */ public MetaStringReader getMetaStringReader() { return metaStringReader; diff --git a/java/fory-core/src/main/java/org/apache/fory/context/RefReader.java b/java/fory-core/src/main/java/org/apache/fory/context/RefReader.java index f18b52973b..c295bedc38 100644 --- a/java/fory-core/src/main/java/org/apache/fory/context/RefReader.java +++ b/java/fory-core/src/main/java/org/apache/fory/context/RefReader.java @@ -49,11 +49,6 @@ public interface RefReader { /** Binds the most recently preserved reference id to {@code object}. */ void reference(Object object); - /** Binds a specific preserved reference id to {@code object}. */ - default void reference(int refId, Object object) { - reference(object); - } - /** Returns the previously materialized object for a specific ref id. */ Object getReadRef(int id); @@ -63,27 +58,6 @@ default void reference(int refId, Object object) { /** Replaces the object stored for a previously preserved ref id. */ void setReadRef(int id, Object object); - /** Starts tracking unresolved reads of the currently constructed object's ref id. */ - default void trackUnresolvedRef(int id) {} - - /** Returns whether a constructor-bound object ref id is currently tracked. */ - default boolean hasTrackedRef() { - return false; - } - - /** Returns the most recently tracked constructor-bound object ref id. */ - default int currentTrackedRefId() { - return -1; - } - - /** Stops tracking unresolved reads for the most recent constructor-bound object. */ - default void untrackUnresolvedRef() {} - - /** Returns and clears whether {@code id} was read before it was bound to an object. */ - default boolean consumeUnresolvedRef(int id) { - return false; - } - /** Clears all per-operation ref-tracking state. */ void reset(); @@ -122,9 +96,6 @@ public boolean hasPreservedRefId() { @Override public void reference(Object object) {} - @Override - public void reference(int refId, Object object) {} - @Override public Object getReadRef(int id) { return null; diff --git a/java/fory-core/src/main/java/org/apache/fory/memory/MemoryUtils.java b/java/fory-core/src/main/java/org/apache/fory/memory/MemoryUtils.java index 99eda09059..bbcc892205 100644 --- a/java/fory-core/src/main/java/org/apache/fory/memory/MemoryUtils.java +++ b/java/fory-core/src/main/java/org/apache/fory/memory/MemoryUtils.java @@ -27,8 +27,8 @@ /** Memory utils for fory. */ public class MemoryUtils { // JDK25+ internal-field access must be backed by supported access in the multi-release classes. - // When a JDK25+ path needs JDK private fields, open the needed java.base package to both - // org.apache.fory.core and org.apache.fory.format. + // The JDK25+ replacement obtains a trusted lookup through java.base/java.lang.invoke instead of + // requiring per-package JDK opens or jdk.unsupported. public static final boolean JDK_INTERNAL_FIELD_ACCESS = !AndroidSupport.IS_ANDROID && !GraalvmSupport.IN_GRAALVM_NATIVE_IMAGE diff --git a/java/fory-core/src/main/java/org/apache/fory/platform/internal/_JDKAccess.java b/java/fory-core/src/main/java/org/apache/fory/platform/internal/_JDKAccess.java index 1c971802c2..54a7259aa5 100644 --- a/java/fory-core/src/main/java/org/apache/fory/platform/internal/_JDKAccess.java +++ b/java/fory-core/src/main/java/org/apache/fory/platform/internal/_JDKAccess.java @@ -359,6 +359,7 @@ private static class SerializationMethods { private static final Method WRITE_OBJECT; private static final Method READ_OBJECT; private static final Method READ_OBJECT_NO_DATA; + private static final Method DEFAULT_READ_OBJECT; private static final Method WRITE_REPLACE; private static final Method READ_RESOLVE; @@ -367,6 +368,7 @@ private static class SerializationMethods { Method writeObject = null; Method readObject = null; Method readObjectNoData = null; + Method defaultReadObject = null; Method writeReplace = null; Method readResolve = null; try { @@ -377,6 +379,12 @@ private static class SerializationMethods { readObject = factoryClass.getDeclaredMethod("readObjectForSerialization", Class.class); readObjectNoData = factoryClass.getDeclaredMethod("readObjectNoDataForSerialization", Class.class); + try { + defaultReadObject = + factoryClass.getDeclaredMethod("defaultReadObjectForSerialization", Class.class); + } catch (NoSuchMethodException e) { + ExceptionUtils.ignore(e); + } writeReplace = factoryClass.getDeclaredMethod("writeReplaceForSerialization", Class.class); readResolve = factoryClass.getDeclaredMethod("readResolveForSerialization", Class.class); } catch (Throwable e) { @@ -386,25 +394,29 @@ private static class SerializationMethods { WRITE_OBJECT = writeObject; READ_OBJECT = readObject; READ_OBJECT_NO_DATA = readObjectNoData; + DEFAULT_READ_OBJECT = defaultReadObject; WRITE_REPLACE = writeReplace; READ_RESOLVE = readResolve; } } - private static Method getSerializationMethod(Class type, Method factoryMethod) { - if (!isSerializationHookLookupAvailable() || factoryMethod == null) { + private static MethodHandle getSerializationHandle(Class type, Method factoryMethod) { + if (SerializationMethods.REFLECTION_FACTORY == null || factoryMethod == null) { return null; } try { - MethodHandle handle = - (MethodHandle) factoryMethod.invoke(SerializationMethods.REFLECTION_FACTORY, type); - return handle == null ? null : MethodHandles.reflectAs(Method.class, handle); + return (MethodHandle) factoryMethod.invoke(SerializationMethods.REFLECTION_FACTORY, type); } catch (Throwable e) { ExceptionUtils.ignore(e); return null; } } + private static Method getSerializationMethod(Class type, Method factoryMethod) { + MethodHandle handle = getSerializationHandle(type, factoryMethod); + return handle == null ? null : MethodHandles.reflectAs(Method.class, handle); + } + public static Method getSerializationWriteObjectMethod(Class type) { return getSerializationMethod(type, SerializationMethods.WRITE_OBJECT); } @@ -417,6 +429,10 @@ public static Method getSerializationReadObjectNoDataMethod(Class type) { return getSerializationMethod(type, SerializationMethods.READ_OBJECT_NO_DATA); } + public static MethodHandle getSerializationDefaultReadObjectHandle(Class type) { + return getSerializationHandle(type, SerializationMethods.DEFAULT_READ_OBJECT); + } + public static Method getSerializationWriteReplaceMethod(Class type) { return getSerializationMethod(type, SerializationMethods.WRITE_REPLACE); } diff --git a/java/fory-core/src/main/java/org/apache/fory/reflect/FieldAccessor.java b/java/fory-core/src/main/java/org/apache/fory/reflect/FieldAccessor.java index 5aa6ebee5a..bfde5ca4fe 100644 --- a/java/fory-core/src/main/java/org/apache/fory/reflect/FieldAccessor.java +++ b/java/fory-core/src/main/java/org/apache/fory/reflect/FieldAccessor.java @@ -19,40 +19,12 @@ package org.apache.fory.reflect; -import java.lang.invoke.MethodHandle; -import java.lang.invoke.MethodHandles; import java.lang.reflect.Field; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; -import java.lang.reflect.Modifier; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.function.Function; -import java.util.function.Predicate; -import java.util.function.ToDoubleFunction; -import java.util.function.ToIntFunction; -import java.util.function.ToLongFunction; -import org.apache.fory.collection.ClassValueCache; -import org.apache.fory.collection.Tuple2; -import org.apache.fory.exception.ForyException; -import org.apache.fory.platform.AndroidSupport; -import org.apache.fory.platform.GraalvmSupport; -import org.apache.fory.platform.internal._JDKAccess; -import org.apache.fory.type.TypeUtils; import org.apache.fory.util.Preconditions; -import org.apache.fory.util.function.Functions; -import org.apache.fory.util.function.ToByteFunction; -import org.apache.fory.util.function.ToCharFunction; -import org.apache.fory.util.function.ToFloatFunction; -import org.apache.fory.util.function.ToShortFunction; -import org.apache.fory.util.record.RecordUtils; -import sun.misc.Unsafe; /** Field accessor for primitive types and object types. */ @SuppressWarnings({"unchecked", "rawtypes"}) public abstract class FieldAccessor { - private static final Unsafe UNSAFE = AndroidSupport.IS_ANDROID ? null : _JDKAccess.UNSAFE; - private static final int REFLECTIVE_ACCESS = 0; private static final int BOOLEAN_ACCESS = 1; private static final int BYTE_ACCESS = 2; private static final int CHAR_ACCESS = 3; @@ -64,37 +36,15 @@ public abstract class FieldAccessor { private static final int OBJECT_ACCESS = 9; protected final Field field; - protected final long fieldOffset; private final int accessKind; public FieldAccessor(Field field) { this.field = field; Preconditions.checkNotNull(field); - this.fieldOffset = fieldOffset(field); - this.accessKind = accessKind(field, fieldOffset); + accessKind = accessKind(field); } - private static long fieldOffset(Field field) { - if (AndroidSupport.IS_ANDROID) { - return -1; - } - if (GraalvmSupport.isGraalBuildTime()) { - // Field offsets are rewritten by GraalVM and are not stable during native-image build time. - return -1; - } - return UNSAFE.objectFieldOffset(field); - } - - protected FieldAccessor(Field field, long fieldOffset) { - this.field = field; - this.fieldOffset = fieldOffset; - this.accessKind = accessKind(field, fieldOffset); - } - - private static int accessKind(Field field, long fieldOffset) { - if (fieldOffset == -1) { - return REFLECTIVE_ACCESS; - } + private static int accessKind(Field field) { Class fieldType = field.getType(); if (fieldType == boolean.class) { return BOOLEAN_ACCESS; @@ -122,46 +72,39 @@ public void set(Object obj, Object value) { throw new UnsupportedOperationException("Unsupported for field " + field); } - public final void copy(Object sourceObject, Object targetObject) { + public void copy(Object sourceObject, Object targetObject) { switch (accessKind) { case BOOLEAN_ACCESS: - UNSAFE.putBoolean(targetObject, fieldOffset, UNSAFE.getBoolean(sourceObject, fieldOffset)); + putBoolean(targetObject, getBoolean(sourceObject)); return; case BYTE_ACCESS: - UNSAFE.putByte(targetObject, fieldOffset, UNSAFE.getByte(sourceObject, fieldOffset)); + putByte(targetObject, getByte(sourceObject)); return; case CHAR_ACCESS: - UNSAFE.putChar(targetObject, fieldOffset, UNSAFE.getChar(sourceObject, fieldOffset)); + putChar(targetObject, getChar(sourceObject)); return; case SHORT_ACCESS: - UNSAFE.putShort(targetObject, fieldOffset, UNSAFE.getShort(sourceObject, fieldOffset)); + putShort(targetObject, getShort(sourceObject)); return; case INT_ACCESS: - UNSAFE.putInt(targetObject, fieldOffset, UNSAFE.getInt(sourceObject, fieldOffset)); + putInt(targetObject, getInt(sourceObject)); return; case LONG_ACCESS: - UNSAFE.putLong(targetObject, fieldOffset, UNSAFE.getLong(sourceObject, fieldOffset)); + putLong(targetObject, getLong(sourceObject)); return; case FLOAT_ACCESS: - UNSAFE.putFloat(targetObject, fieldOffset, UNSAFE.getFloat(sourceObject, fieldOffset)); + putFloat(targetObject, getFloat(sourceObject)); return; case DOUBLE_ACCESS: - UNSAFE.putDouble(targetObject, fieldOffset, UNSAFE.getDouble(sourceObject, fieldOffset)); - return; - case OBJECT_ACCESS: - UNSAFE.putObject(targetObject, fieldOffset, UNSAFE.getObject(sourceObject, fieldOffset)); + putDouble(targetObject, getDouble(sourceObject)); return; default: putObject(targetObject, getObject(sourceObject)); } } - public final void copyObject(Object sourceObject, Object targetObject) { - if (accessKind == OBJECT_ACCESS) { - UNSAFE.putObject(targetObject, fieldOffset, UNSAFE.getObject(sourceObject, fieldOffset)); - } else { - putObject(targetObject, getObject(sourceObject)); - } + public void copyObject(Object sourceObject, Object targetObject) { + putObject(targetObject, getObject(sourceObject)); } public Field getField() { @@ -232,25 +175,12 @@ public void putDouble(Object targetObject, double value) { set(targetObject, value); } - public final void putObject(Object targetObject, Object object) { - // For primitive fields, we must use set() which calls the correct UNSAFE.putXxx method. - // UNSAFE.putObject writes object references, not primitive values. - if (fieldOffset != -1 && !field.getType().isPrimitive()) { - UNSAFE.putObject(targetObject, fieldOffset, object); - } else { - set(targetObject, object); - } + public void putObject(Object targetObject, Object object) { + set(targetObject, object); } - public final Object getObject(Object targetObject) { - // For primitive fields, we must use get() which calls the correct UNSAFE.getXxx method - // and returns the boxed value. UNSAFE.getObject interprets primitive bytes as object - // refs. - if (fieldOffset != -1 && !field.getType().isPrimitive()) { - return UNSAFE.getObject(targetObject, fieldOffset); - } else { - return get(targetObject); - } + public Object getObject(Object targetObject) { + return get(targetObject); } void checkObj(Object obj) { @@ -265,11 +195,10 @@ public String toString() { } public abstract static class FieldGetter extends FieldAccessor { - private final Object getter; protected FieldGetter(Field field, Object getter) { - super(field, -1); + super(field); this.getter = getter; } @@ -279,784 +208,10 @@ public Object getGetter() { } public static FieldAccessor createAccessor(Field field) { - Preconditions.checkArgument(!Modifier.isStatic(field.getModifiers()), field); - if (RecordUtils.isRecord(field.getDeclaringClass())) { - if (AndroidSupport.IS_ANDROID || GraalvmSupport.IN_GRAALVM_NATIVE_IMAGE) { - return new ReflectiveRecordFieldAccessor(field); - } - Object getter; - try { - Method getterMethod = field.getDeclaringClass().getDeclaredMethod(field.getName()); - getter = Functions.makeGetterFunction(getterMethod); - } catch (NoSuchMethodException ex) { - throw new RuntimeException(ex); - } - if (getter instanceof Predicate) { - return new BooleanGetter(field, (Predicate) getter); - } else if (getter instanceof ToByteFunction) { - return new ByteGetter(field, (ToByteFunction) getter); - } else if (getter instanceof ToCharFunction) { - return new CharGetter(field, (ToCharFunction) getter); - } else if (getter instanceof ToShortFunction) { - return new ShortGetter(field, (ToShortFunction) getter); - } else if (getter instanceof ToIntFunction) { - return new IntGetter(field, (ToIntFunction) getter); - } else if (getter instanceof ToLongFunction) { - return new LongGetter(field, (ToLongFunction) getter); - } else if (getter instanceof ToFloatFunction) { - return new FloatGetter(field, (ToFloatFunction) getter); - } else if (getter instanceof ToDoubleFunction) { - return new DoubleGetter(field, (ToDoubleFunction) getter); - } else { - return new ObjectGetter(field, (Function) getter); - } - } - if (AndroidSupport.IS_ANDROID) { - // Android field access must stay reflection-owned: no Unsafe offsets, trusted lookups, - // generated accessors, or primitive-specific reflection subclasses. - return new ReflectionFieldAccessor(field); - } - if (GraalvmSupport.isGraalBuildTime()) { - return new GeneratedAccessor(field); - } - if (field.getType() == boolean.class) { - return new BooleanAccessor(field); - } else if (field.getType() == byte.class) { - return new ByteAccessor(field); - } else if (field.getType() == char.class) { - return new CharAccessor(field); - } else if (field.getType() == short.class) { - return new ShortAccessor(field); - } else if (field.getType() == int.class) { - return new IntAccessor(field); - } else if (field.getType() == long.class) { - return new LongAccessor(field); - } else if (field.getType() == float.class) { - return new FloatAccessor(field); - } else if (field.getType() == double.class) { - return new DoubleAccessor(field); - } else { - return new ObjectAccessor(field); - } + return FieldAccessorFactory.createAccessor(field); } public static FieldAccessor createStaticAccessor(Field field) { - Preconditions.checkArgument(Modifier.isStatic(field.getModifiers()), field); - if (AndroidSupport.IS_ANDROID) { - field.setAccessible(true); - return new ReflectiveStaticFieldAccessor(field); - } - return new StaticObjectAccessor(field); - } - - static final class ReflectiveRecordFieldAccessor extends FieldGetter { - private final Method accessor; - - ReflectiveRecordFieldAccessor(Field field) { - super(field, null); - try { - accessor = field.getDeclaringClass().getDeclaredMethod(field.getName()); - accessor.setAccessible(true); - } catch (NoSuchMethodException | RuntimeException e) { - throw new ForyException("Failed to create record field accessor for " + field, e); - } - } - - @Override - public Object get(Object obj) { - checkObj(obj); - try { - return accessor.invoke(obj); - } catch (IllegalAccessException | IllegalArgumentException e) { - throw new ForyException("Failed to read record field reflectively: " + field, e); - } catch (InvocationTargetException e) { - throw new ForyException( - "Record accessor threw while reading field: " + field, e.getCause()); - } - } - - @Override - public void set(Object obj, Object value) { - throw new UnsupportedOperationException("Record field is read-only: " + field); - } - } - - /** Primitive boolean accessor. */ - public static class BooleanAccessor extends FieldAccessor { - public BooleanAccessor(Field field) { - super(field); - Preconditions.checkArgument(field.getType() == boolean.class); - } - - @Override - public Object get(Object obj) { - return getBoolean(obj); - } - - @Override - public boolean getBoolean(Object obj) { - checkObj(obj); - return UNSAFE.getBoolean(obj, fieldOffset); - } - - @Override - public void set(Object obj, Object value) { - putBoolean(obj, (Boolean) value); - } - - @Override - public void putBoolean(Object obj, boolean value) { - checkObj(obj); - UNSAFE.putBoolean(obj, fieldOffset, value); - } - } - - public static class BooleanGetter extends FieldGetter { - private final Predicate getter; - - public BooleanGetter(Field field, Predicate getter) { - super(field, getter); - this.getter = getter; - Preconditions.checkArgument(field.getType() == boolean.class); - } - - @Override - public Boolean get(Object obj) { - return getBoolean(obj); - } - - @Override - public boolean getBoolean(Object obj) { - checkObj(obj); - return getter.test(obj); - } - } - - /** Primitive byte accessor. */ - public static class ByteAccessor extends FieldAccessor { - public ByteAccessor(Field field) { - super(field); - Preconditions.checkArgument(field.getType() == byte.class); - } - - @Override - public Byte get(Object obj) { - return getByte(obj); - } - - @Override - public byte getByte(Object obj) { - checkObj(obj); - return UNSAFE.getByte(obj, fieldOffset); - } - - @Override - public void set(Object obj, Object value) { - putByte(obj, (Byte) value); - } - - @Override - public void putByte(Object obj, byte value) { - checkObj(obj); - UNSAFE.putByte(obj, fieldOffset, value); - } - } - - public static class ByteGetter extends FieldGetter { - - private final ToByteFunction getter; - - public ByteGetter(Field field, ToByteFunction getter) { - super(field, getter); - this.getter = getter; - Preconditions.checkArgument(field.getType() == byte.class); - } - - @Override - public Byte get(Object obj) { - return getByte(obj); - } - - @Override - public byte getByte(Object obj) { - return getter.applyAsByte(obj); - } - } - - /** Primitive char accessor. */ - public static class CharAccessor extends FieldAccessor { - public CharAccessor(Field field) { - super(field); - Preconditions.checkArgument(field.getType() == char.class); - } - - @Override - public Character get(Object obj) { - return getChar(obj); - } - - @Override - public char getChar(Object obj) { - checkObj(obj); - return UNSAFE.getChar(obj, fieldOffset); - } - - @Override - public void set(Object obj, Object value) { - putChar(obj, (Character) value); - } - - @Override - public void putChar(Object obj, char value) { - checkObj(obj); - UNSAFE.putChar(obj, fieldOffset, value); - } - } - - public static class CharGetter extends FieldGetter { - private final ToCharFunction getter; - - public CharGetter(Field field, ToCharFunction getter) { - super(field, getter); - this.getter = getter; - Preconditions.checkArgument(field.getType() == char.class); - } - - @Override - public Character get(Object obj) { - return getChar(obj); - } - - @Override - public char getChar(Object obj) { - return getter.applyAsChar(obj); - } - } - - /** Primitive short accessor. */ - public static class ShortAccessor extends FieldAccessor { - public ShortAccessor(Field field) { - super(field); - Preconditions.checkArgument(field.getType() == short.class); - } - - @Override - public Short get(Object obj) { - return getShort(obj); - } - - @Override - public short getShort(Object obj) { - checkObj(obj); - return UNSAFE.getShort(obj, fieldOffset); - } - - @Override - public void set(Object obj, Object value) { - putShort(obj, (Short) value); - } - - @Override - public void putShort(Object obj, short value) { - checkObj(obj); - UNSAFE.putShort(obj, fieldOffset, value); - } - } - - public static class ShortGetter extends FieldGetter { - private final ToShortFunction getter; - - public ShortGetter(Field field, ToShortFunction getter) { - super(field, getter); - this.getter = getter; - Preconditions.checkArgument(field.getType() == short.class); - } - - @Override - public Short get(Object obj) { - return getShort(obj); - } - - @Override - public short getShort(Object obj) { - return getter.applyAsShort(obj); - } - } - - /** Primitive int accessor. */ - public static class IntAccessor extends FieldAccessor { - public IntAccessor(Field field) { - super(field); - Preconditions.checkArgument(field.getType() == int.class); - } - - @Override - public Integer get(Object obj) { - return getInt(obj); - } - - @Override - public int getInt(Object obj) { - checkObj(obj); - return UNSAFE.getInt(obj, fieldOffset); - } - - @Override - public void set(Object obj, Object value) { - putInt(obj, (Integer) value); - } - - @Override - public void putInt(Object obj, int value) { - checkObj(obj); - UNSAFE.putInt(obj, fieldOffset, value); - } - } - - public static class IntGetter extends FieldGetter { - private final ToIntFunction getter; - - public IntGetter(Field field, ToIntFunction getter) { - super(field, getter); - this.getter = getter; - Preconditions.checkArgument(field.getType() == int.class); - } - - @Override - public Integer get(Object obj) { - return getInt(obj); - } - - @Override - public int getInt(Object obj) { - return getter.applyAsInt(obj); - } - } - - /** Primitive long accessor. */ - public static class LongAccessor extends FieldAccessor { - public LongAccessor(Field field) { - super(field); - Preconditions.checkArgument(field.getType() == long.class); - } - - @Override - public Long get(Object obj) { - return getLong(obj); - } - - @Override - public long getLong(Object obj) { - checkObj(obj); - return UNSAFE.getLong(obj, fieldOffset); - } - - @Override - public void set(Object obj, Object value) { - putLong(obj, (Long) value); - } - - @Override - public void putLong(Object obj, long value) { - checkObj(obj); - UNSAFE.putLong(obj, fieldOffset, value); - } - } - - public static class LongGetter extends FieldGetter { - private final ToLongFunction getter; - - public LongGetter(Field field, ToLongFunction getter) { - super(field, getter); - this.getter = getter; - Preconditions.checkArgument(field.getType() == long.class); - } - - @Override - public Long get(Object obj) { - return getLong(obj); - } - - @Override - public long getLong(Object obj) { - return getter.applyAsLong(obj); - } - } - - /** Primitive float accessor. */ - public static class FloatAccessor extends FieldAccessor { - public FloatAccessor(Field field) { - super(field); - Preconditions.checkArgument(field.getType() == float.class); - } - - @Override - public Object get(Object obj) { - return getFloat(obj); - } - - @Override - public float getFloat(Object obj) { - checkObj(obj); - return UNSAFE.getFloat(obj, fieldOffset); - } - - @Override - public void set(Object obj, Object value) { - putFloat(obj, (Float) value); - } - - @Override - public void putFloat(Object obj, float value) { - checkObj(obj); - UNSAFE.putFloat(obj, fieldOffset, value); - } - } - - public static class FloatGetter extends FieldGetter { - private final ToFloatFunction getter; - - public FloatGetter(Field field, ToFloatFunction getter) { - super(field, getter); - this.getter = getter; - Preconditions.checkArgument(field.getType() == float.class); - } - - @Override - public Float get(Object obj) { - return getFloat(obj); - } - - @Override - public float getFloat(Object obj) { - return getter.applyAsFloat(obj); - } - } - - /** Primitive double accessor. */ - public static class DoubleAccessor extends FieldAccessor { - public DoubleAccessor(Field field) { - super(field); - Preconditions.checkArgument(field.getType() == double.class); - } - - @Override - public Object get(Object obj) { - return getDouble(obj); - } - - @Override - public double getDouble(Object obj) { - checkObj(obj); - return UNSAFE.getDouble(obj, fieldOffset); - } - - @Override - public void set(Object obj, Object value) { - putDouble(obj, (Double) value); - } - - @Override - public void putDouble(Object obj, double value) { - checkObj(obj); - UNSAFE.putDouble(obj, fieldOffset, value); - } - } - - public static class DoubleGetter extends FieldGetter { - private final ToDoubleFunction getter; - - public DoubleGetter(Field field, ToDoubleFunction getter) { - super(field, getter); - this.getter = getter; - Preconditions.checkArgument(field.getType() == double.class); - } - - @Override - public Double get(Object obj) { - return getDouble(obj); - } - - @Override - public double getDouble(Object obj) { - return getter.applyAsDouble(obj); - } - } - - /** Object accessor. */ - public static class ObjectAccessor extends FieldAccessor { - public ObjectAccessor(Field field) { - super(field); - Preconditions.checkArgument(!TypeUtils.isPrimitive(field.getType())); - } - - @Override - public Object get(Object obj) { - checkObj(obj); - return UNSAFE.getObject(obj, fieldOffset); - } - - @Override - public void set(Object obj, Object value) { - checkObj(obj); - UNSAFE.putObject(obj, fieldOffset, value); - } - } - - public static class ObjectGetter extends FieldGetter { - private final Function getter; - - public ObjectGetter(Field field, Function getter) { - super(field, getter); - this.getter = getter; - Preconditions.checkArgument(!field.getType().isPrimitive(), field); - } - - @Override - public Object get(Object obj) { - return getter.apply(obj); - } - } - - static final class ReflectiveStaticFieldAccessor extends FieldAccessor { - ReflectiveStaticFieldAccessor(Field field) { - super(field, -1); - } - - @Override - public Object get(Object obj) { - try { - return field.get(null); - } catch (IllegalAccessException | IllegalArgumentException e) { - throw new ForyException("Failed to read static field reflectively: " + field, e); - } - } - - @Override - public void set(Object obj, Object value) { - try { - field.set(null, value); - } catch (IllegalAccessException | IllegalArgumentException e) { - throw new ForyException("Failed to write static field reflectively: " + field, e); - } - } - } - - static final class StaticObjectAccessor extends FieldAccessor { - private final Object base; - private final long offset; - - StaticObjectAccessor(Field field) { - super(field, -1); - Preconditions.checkArgument(!TypeUtils.isPrimitive(field.getType())); - base = UNSAFE.staticFieldBase(field); - offset = UNSAFE.staticFieldOffset(field); - } - - @Override - public Object get(Object obj) { - return UNSAFE.getObject(base, offset); - } - - @Override - public void set(Object obj, Object value) { - UNSAFE.putObject(base, offset, value); - } - } - - static final class GeneratedAccessor extends FieldAccessor { - private static final ClassValueCache>> - cache = ClassValueCache.newClassKeyCache(8); - - private final MethodHandle getter; - private final MethodHandle setter; - - GeneratedAccessor(Field field) { - super(field, -1); - ConcurrentMap> map = - cache.get(field.getDeclaringClass(), ConcurrentHashMap::new); - ; - MethodHandles.Lookup lookup = _JDKAccess._trustedLookup(field.getDeclaringClass()); - Tuple2 tuple2 = - map.computeIfAbsent( - field.getName(), - k -> { - try { - MethodHandle getter = - lookup.findGetter( - field.getDeclaringClass(), field.getName(), field.getType()); - MethodHandle setter = - lookup.findSetter( - field.getDeclaringClass(), field.getName(), field.getType()); - return Tuple2.of(getter, setter); - } catch (IllegalAccessException | NoSuchFieldException ex) { - throw new RuntimeException(ex); - } - }); - getter = tuple2.f0; - setter = tuple2.f1; - } - - @Override - public Object get(Object obj) { - try { - return getter.invoke(obj); - } catch (Throwable e) { - throw new RuntimeException(e); - } - } - - @Override - public void set(Object obj, Object value) { - try { - setter.invoke(obj, value); - } catch (Throwable e) { - throw new RuntimeException(e); - } - } - - @Override - public boolean getBoolean(Object obj) { - try { - return (boolean) getter.invoke(obj); - } catch (Throwable e) { - throw new RuntimeException(e); - } - } - - @Override - public void putBoolean(Object obj, boolean value) { - try { - setter.invoke(obj, value); - } catch (Throwable e) { - throw new RuntimeException(e); - } - } - - @Override - public byte getByte(Object obj) { - try { - return (byte) getter.invoke(obj); - } catch (Throwable e) { - throw new RuntimeException(e); - } - } - - @Override - public void putByte(Object obj, byte value) { - try { - setter.invoke(obj, value); - } catch (Throwable e) { - throw new RuntimeException(e); - } - } - - @Override - public char getChar(Object obj) { - try { - return (char) getter.invoke(obj); - } catch (Throwable e) { - throw new RuntimeException(e); - } - } - - @Override - public void putChar(Object obj, char value) { - try { - setter.invoke(obj, value); - } catch (Throwable e) { - throw new RuntimeException(e); - } - } - - @Override - public short getShort(Object obj) { - try { - return (short) getter.invoke(obj); - } catch (Throwable e) { - throw new RuntimeException(e); - } - } - - @Override - public void putShort(Object obj, short value) { - try { - setter.invoke(obj, value); - } catch (Throwable e) { - throw new RuntimeException(e); - } - } - - @Override - public int getInt(Object obj) { - try { - return (int) getter.invoke(obj); - } catch (Throwable e) { - throw new RuntimeException(e); - } - } - - @Override - public void putInt(Object obj, int value) { - try { - setter.invoke(obj, value); - } catch (Throwable e) { - throw new RuntimeException(e); - } - } - - @Override - public long getLong(Object obj) { - try { - return (long) getter.invoke(obj); - } catch (Throwable e) { - throw new RuntimeException(e); - } - } - - @Override - public void putLong(Object obj, long value) { - try { - setter.invoke(obj, value); - } catch (Throwable e) { - throw new RuntimeException(e); - } - } - - @Override - public float getFloat(Object obj) { - try { - return (float) getter.invoke(obj); - } catch (Throwable e) { - throw new RuntimeException(e); - } - } - - @Override - public void putFloat(Object obj, float value) { - try { - setter.invoke(obj, value); - } catch (Throwable e) { - throw new RuntimeException(e); - } - } - - @Override - public double getDouble(Object obj) { - try { - return (double) getter.invoke(obj); - } catch (Throwable e) { - throw new RuntimeException(e); - } - } - - @Override - public void putDouble(Object obj, double value) { - try { - setter.invoke(obj, value); - } catch (Throwable e) { - throw new RuntimeException(e); - } - } + return FieldAccessorFactory.createStaticAccessor(field); } } diff --git a/java/fory-core/src/main/java/org/apache/fory/reflect/FieldAccessorFactory.java b/java/fory-core/src/main/java/org/apache/fory/reflect/FieldAccessorFactory.java new file mode 100644 index 0000000000..7f1bc6c684 --- /dev/null +++ b/java/fory-core/src/main/java/org/apache/fory/reflect/FieldAccessorFactory.java @@ -0,0 +1,331 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.reflect; + +import java.lang.reflect.Field; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.lang.reflect.Modifier; +import java.util.function.Function; +import java.util.function.Predicate; +import java.util.function.ToDoubleFunction; +import java.util.function.ToIntFunction; +import java.util.function.ToLongFunction; +import org.apache.fory.exception.ForyException; +import org.apache.fory.platform.AndroidSupport; +import org.apache.fory.platform.GraalvmSupport; +import org.apache.fory.reflect.FieldAccessor.FieldGetter; +import org.apache.fory.util.Preconditions; +import org.apache.fory.util.function.Functions; +import org.apache.fory.util.function.ToByteFunction; +import org.apache.fory.util.function.ToCharFunction; +import org.apache.fory.util.function.ToFloatFunction; +import org.apache.fory.util.function.ToShortFunction; +import org.apache.fory.util.record.RecordUtils; + +final class FieldAccessorFactory { + private FieldAccessorFactory() {} + + static FieldAccessor createAccessor(Field field) { + Preconditions.checkArgument(!Modifier.isStatic(field.getModifiers()), field); + if (RecordUtils.isRecord(field.getDeclaringClass())) { + return createRecordAccessor(field); + } + if (AndroidSupport.IS_ANDROID) { + // Android field access must stay reflection-owned: no Unsafe offsets, trusted lookups, + // generated accessors, or primitive-specific reflection subclasses. + return new ReflectionFieldAccessor(field); + } + return FieldAccessorStrategy.createAccessor(field); + } + + static FieldAccessor createStaticAccessor(Field field) { + Preconditions.checkArgument(Modifier.isStatic(field.getModifiers()), field); + if (AndroidSupport.IS_ANDROID) { + field.setAccessible(true); + return new ReflectiveStaticFieldAccessor(field); + } + return FieldAccessorStrategy.createStaticAccessor(field); + } + + private static FieldAccessor createRecordAccessor(Field field) { + if (AndroidSupport.IS_ANDROID || GraalvmSupport.IN_GRAALVM_NATIVE_IMAGE) { + return new ReflectiveRecordFieldAccessor(field); + } + Object getter; + try { + Method getterMethod = field.getDeclaringClass().getDeclaredMethod(field.getName()); + getter = Functions.makeGetterFunction(getterMethod); + } catch (NoSuchMethodException ex) { + throw new RuntimeException(ex); + } + if (getter instanceof Predicate) { + return new BooleanGetter(field, (Predicate) getter); + } else if (getter instanceof ToByteFunction) { + return new ByteGetter(field, (ToByteFunction) getter); + } else if (getter instanceof ToCharFunction) { + return new CharGetter(field, (ToCharFunction) getter); + } else if (getter instanceof ToShortFunction) { + return new ShortGetter(field, (ToShortFunction) getter); + } else if (getter instanceof ToIntFunction) { + return new IntGetter(field, (ToIntFunction) getter); + } else if (getter instanceof ToLongFunction) { + return new LongGetter(field, (ToLongFunction) getter); + } else if (getter instanceof ToFloatFunction) { + return new FloatGetter(field, (ToFloatFunction) getter); + } else if (getter instanceof ToDoubleFunction) { + return new DoubleGetter(field, (ToDoubleFunction) getter); + } else { + return new ObjectGetter(field, (Function) getter); + } + } + + static final class ReflectiveRecordFieldAccessor extends FieldGetter { + private final Method accessor; + + ReflectiveRecordFieldAccessor(Field field) { + super(field, null); + try { + accessor = field.getDeclaringClass().getDeclaredMethod(field.getName()); + accessor.setAccessible(true); + } catch (NoSuchMethodException | RuntimeException e) { + throw new ForyException("Failed to create record field accessor for " + field, e); + } + } + + @Override + public Object get(Object obj) { + checkObj(obj); + try { + return accessor.invoke(obj); + } catch (IllegalAccessException | IllegalArgumentException e) { + throw new ForyException("Failed to read record field reflectively: " + field, e); + } catch (InvocationTargetException e) { + throw new ForyException( + "Record accessor threw while reading field: " + field, e.getCause()); + } + } + + @Override + public void set(Object obj, Object value) { + throw new UnsupportedOperationException("Record field is read-only: " + field); + } + } + + public static class BooleanGetter extends FieldGetter { + private final Predicate getter; + + public BooleanGetter(Field field, Predicate getter) { + super(field, getter); + this.getter = getter; + Preconditions.checkArgument(field.getType() == boolean.class); + } + + @Override + public Boolean get(Object obj) { + return getBoolean(obj); + } + + @Override + public boolean getBoolean(Object obj) { + checkObj(obj); + return getter.test(obj); + } + } + + public static class ByteGetter extends FieldGetter { + private final ToByteFunction getter; + + public ByteGetter(Field field, ToByteFunction getter) { + super(field, getter); + this.getter = getter; + Preconditions.checkArgument(field.getType() == byte.class); + } + + @Override + public Byte get(Object obj) { + return getByte(obj); + } + + @Override + public byte getByte(Object obj) { + return getter.applyAsByte(obj); + } + } + + public static class CharGetter extends FieldGetter { + private final ToCharFunction getter; + + public CharGetter(Field field, ToCharFunction getter) { + super(field, getter); + this.getter = getter; + Preconditions.checkArgument(field.getType() == char.class); + } + + @Override + public Character get(Object obj) { + return getChar(obj); + } + + @Override + public char getChar(Object obj) { + return getter.applyAsChar(obj); + } + } + + public static class ShortGetter extends FieldGetter { + private final ToShortFunction getter; + + public ShortGetter(Field field, ToShortFunction getter) { + super(field, getter); + this.getter = getter; + Preconditions.checkArgument(field.getType() == short.class); + } + + @Override + public Short get(Object obj) { + return getShort(obj); + } + + @Override + public short getShort(Object obj) { + return getter.applyAsShort(obj); + } + } + + public static class IntGetter extends FieldGetter { + private final ToIntFunction getter; + + public IntGetter(Field field, ToIntFunction getter) { + super(field, getter); + this.getter = getter; + Preconditions.checkArgument(field.getType() == int.class); + } + + @Override + public Integer get(Object obj) { + return getInt(obj); + } + + @Override + public int getInt(Object obj) { + return getter.applyAsInt(obj); + } + } + + public static class LongGetter extends FieldGetter { + private final ToLongFunction getter; + + public LongGetter(Field field, ToLongFunction getter) { + super(field, getter); + this.getter = getter; + Preconditions.checkArgument(field.getType() == long.class); + } + + @Override + public Long get(Object obj) { + return getLong(obj); + } + + @Override + public long getLong(Object obj) { + return getter.applyAsLong(obj); + } + } + + public static class FloatGetter extends FieldGetter { + private final ToFloatFunction getter; + + public FloatGetter(Field field, ToFloatFunction getter) { + super(field, getter); + this.getter = getter; + Preconditions.checkArgument(field.getType() == float.class); + } + + @Override + public Float get(Object obj) { + return getFloat(obj); + } + + @Override + public float getFloat(Object obj) { + return getter.applyAsFloat(obj); + } + } + + public static class DoubleGetter extends FieldGetter { + private final ToDoubleFunction getter; + + public DoubleGetter(Field field, ToDoubleFunction getter) { + super(field, getter); + this.getter = getter; + Preconditions.checkArgument(field.getType() == double.class); + } + + @Override + public Double get(Object obj) { + return getDouble(obj); + } + + @Override + public double getDouble(Object obj) { + return getter.applyAsDouble(obj); + } + } + + public static class ObjectGetter extends FieldGetter { + private final Function getter; + + public ObjectGetter(Field field, Function getter) { + super(field, getter); + this.getter = getter; + Preconditions.checkArgument(!field.getType().isPrimitive(), field); + } + + @Override + public Object get(Object obj) { + return getter.apply(obj); + } + } + + static final class ReflectiveStaticFieldAccessor extends FieldAccessor { + ReflectiveStaticFieldAccessor(Field field) { + super(field); + } + + @Override + public Object get(Object obj) { + try { + return field.get(null); + } catch (IllegalAccessException | IllegalArgumentException e) { + throw new ForyException("Failed to read static field reflectively: " + field, e); + } + } + + @Override + public void set(Object obj, Object value) { + try { + field.set(null, value); + } catch (IllegalAccessException | IllegalArgumentException e) { + throw new ForyException("Failed to write static field reflectively: " + field, e); + } + } + } +} diff --git a/java/fory-core/src/main/java/org/apache/fory/reflect/FieldAccessorStrategy.java b/java/fory-core/src/main/java/org/apache/fory/reflect/FieldAccessorStrategy.java new file mode 100644 index 0000000000..22a503cf90 --- /dev/null +++ b/java/fory-core/src/main/java/org/apache/fory/reflect/FieldAccessorStrategy.java @@ -0,0 +1,648 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.reflect; + +import java.lang.invoke.MethodHandle; +import java.lang.invoke.MethodHandles; +import java.lang.reflect.Field; +import java.lang.reflect.Modifier; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import org.apache.fory.collection.ClassValueCache; +import org.apache.fory.collection.Tuple2; +import org.apache.fory.platform.AndroidSupport; +import org.apache.fory.platform.GraalvmSupport; +import org.apache.fory.platform.internal._JDKAccess; +import org.apache.fory.type.TypeUtils; +import org.apache.fory.util.Preconditions; +import sun.misc.Unsafe; + +final class FieldAccessorStrategy { + private static final Unsafe UNSAFE = AndroidSupport.IS_ANDROID ? null : _JDKAccess.UNSAFE; + + private static final int BOOLEAN_ACCESS = 1; + private static final int BYTE_ACCESS = 2; + private static final int CHAR_ACCESS = 3; + private static final int SHORT_ACCESS = 4; + private static final int INT_ACCESS = 5; + private static final int LONG_ACCESS = 6; + private static final int FLOAT_ACCESS = 7; + private static final int DOUBLE_ACCESS = 8; + private static final int OBJECT_ACCESS = 9; + + private FieldAccessorStrategy() {} + + private abstract static class UnsafeAccessor extends FieldAccessor { + protected final long fieldOffset; + private final int accessKind; + + UnsafeAccessor(Field field) { + super(field); + fieldOffset = fieldOffset(field); + accessKind = accessKind(field); + } + + private static long fieldOffset(Field field) { + if (AndroidSupport.IS_ANDROID) { + return -1; + } + if (GraalvmSupport.isGraalBuildTime()) { + // Field offsets are rewritten by GraalVM and are not stable during native-image build time. + return -1; + } + return UNSAFE.objectFieldOffset(field); + } + + private static int accessKind(Field field) { + Class fieldType = field.getType(); + if (fieldType == boolean.class) { + return BOOLEAN_ACCESS; + } else if (fieldType == byte.class) { + return BYTE_ACCESS; + } else if (fieldType == char.class) { + return CHAR_ACCESS; + } else if (fieldType == short.class) { + return SHORT_ACCESS; + } else if (fieldType == int.class) { + return INT_ACCESS; + } else if (fieldType == long.class) { + return LONG_ACCESS; + } else if (fieldType == float.class) { + return FLOAT_ACCESS; + } else if (fieldType == double.class) { + return DOUBLE_ACCESS; + } + return OBJECT_ACCESS; + } + + @Override + public void copy(Object sourceObject, Object targetObject) { + switch (accessKind) { + case BOOLEAN_ACCESS: + UNSAFE.putBoolean( + targetObject, fieldOffset, UNSAFE.getBoolean(sourceObject, fieldOffset)); + return; + case BYTE_ACCESS: + UNSAFE.putByte(targetObject, fieldOffset, UNSAFE.getByte(sourceObject, fieldOffset)); + return; + case CHAR_ACCESS: + UNSAFE.putChar(targetObject, fieldOffset, UNSAFE.getChar(sourceObject, fieldOffset)); + return; + case SHORT_ACCESS: + UNSAFE.putShort(targetObject, fieldOffset, UNSAFE.getShort(sourceObject, fieldOffset)); + return; + case INT_ACCESS: + UNSAFE.putInt(targetObject, fieldOffset, UNSAFE.getInt(sourceObject, fieldOffset)); + return; + case LONG_ACCESS: + UNSAFE.putLong(targetObject, fieldOffset, UNSAFE.getLong(sourceObject, fieldOffset)); + return; + case FLOAT_ACCESS: + UNSAFE.putFloat(targetObject, fieldOffset, UNSAFE.getFloat(sourceObject, fieldOffset)); + return; + case DOUBLE_ACCESS: + UNSAFE.putDouble(targetObject, fieldOffset, UNSAFE.getDouble(sourceObject, fieldOffset)); + return; + case OBJECT_ACCESS: + UNSAFE.putObject(targetObject, fieldOffset, UNSAFE.getObject(sourceObject, fieldOffset)); + return; + default: + putObject(targetObject, getObject(sourceObject)); + } + } + + @Override + public void copyObject(Object sourceObject, Object targetObject) { + if (accessKind == OBJECT_ACCESS) { + UNSAFE.putObject(targetObject, fieldOffset, UNSAFE.getObject(sourceObject, fieldOffset)); + } else { + putObject(targetObject, getObject(sourceObject)); + } + } + } + + static FieldAccessor createAccessor(Field field) { + Preconditions.checkArgument(!Modifier.isStatic(field.getModifiers()), field); + if (GraalvmSupport.isGraalBuildTime()) { + return new GeneratedAccessor(field); + } + if (field.getType() == boolean.class) { + return new BooleanAccessor(field); + } else if (field.getType() == byte.class) { + return new ByteAccessor(field); + } else if (field.getType() == char.class) { + return new CharAccessor(field); + } else if (field.getType() == short.class) { + return new ShortAccessor(field); + } else if (field.getType() == int.class) { + return new IntAccessor(field); + } else if (field.getType() == long.class) { + return new LongAccessor(field); + } else if (field.getType() == float.class) { + return new FloatAccessor(field); + } else if (field.getType() == double.class) { + return new DoubleAccessor(field); + } else { + return new ObjectAccessor(field); + } + } + + static FieldAccessor createStaticAccessor(Field field) { + Preconditions.checkArgument(Modifier.isStatic(field.getModifiers()), field); + return new StaticObjectAccessor(field); + } + + /** Primitive boolean accessor. */ + public static class BooleanAccessor extends UnsafeAccessor { + public BooleanAccessor(Field field) { + super(field); + Preconditions.checkArgument(field.getType() == boolean.class); + } + + @Override + public Object get(Object obj) { + return getBoolean(obj); + } + + @Override + public boolean getBoolean(Object obj) { + checkObj(obj); + return UNSAFE.getBoolean(obj, fieldOffset); + } + + @Override + public void set(Object obj, Object value) { + putBoolean(obj, (Boolean) value); + } + + @Override + public void putBoolean(Object obj, boolean value) { + checkObj(obj); + UNSAFE.putBoolean(obj, fieldOffset, value); + } + } + + /** Primitive byte accessor. */ + public static class ByteAccessor extends UnsafeAccessor { + public ByteAccessor(Field field) { + super(field); + Preconditions.checkArgument(field.getType() == byte.class); + } + + @Override + public Byte get(Object obj) { + return getByte(obj); + } + + @Override + public byte getByte(Object obj) { + checkObj(obj); + return UNSAFE.getByte(obj, fieldOffset); + } + + @Override + public void set(Object obj, Object value) { + putByte(obj, (Byte) value); + } + + @Override + public void putByte(Object obj, byte value) { + checkObj(obj); + UNSAFE.putByte(obj, fieldOffset, value); + } + } + + /** Primitive char accessor. */ + public static class CharAccessor extends UnsafeAccessor { + public CharAccessor(Field field) { + super(field); + Preconditions.checkArgument(field.getType() == char.class); + } + + @Override + public Character get(Object obj) { + return getChar(obj); + } + + @Override + public char getChar(Object obj) { + checkObj(obj); + return UNSAFE.getChar(obj, fieldOffset); + } + + @Override + public void set(Object obj, Object value) { + putChar(obj, (Character) value); + } + + @Override + public void putChar(Object obj, char value) { + checkObj(obj); + UNSAFE.putChar(obj, fieldOffset, value); + } + } + + /** Primitive short accessor. */ + public static class ShortAccessor extends UnsafeAccessor { + public ShortAccessor(Field field) { + super(field); + Preconditions.checkArgument(field.getType() == short.class); + } + + @Override + public Short get(Object obj) { + return getShort(obj); + } + + @Override + public short getShort(Object obj) { + checkObj(obj); + return UNSAFE.getShort(obj, fieldOffset); + } + + @Override + public void set(Object obj, Object value) { + putShort(obj, (Short) value); + } + + @Override + public void putShort(Object obj, short value) { + checkObj(obj); + UNSAFE.putShort(obj, fieldOffset, value); + } + } + + /** Primitive int accessor. */ + public static class IntAccessor extends UnsafeAccessor { + public IntAccessor(Field field) { + super(field); + Preconditions.checkArgument(field.getType() == int.class); + } + + @Override + public Integer get(Object obj) { + return getInt(obj); + } + + @Override + public int getInt(Object obj) { + checkObj(obj); + return UNSAFE.getInt(obj, fieldOffset); + } + + @Override + public void set(Object obj, Object value) { + putInt(obj, (Integer) value); + } + + @Override + public void putInt(Object obj, int value) { + checkObj(obj); + UNSAFE.putInt(obj, fieldOffset, value); + } + } + + /** Primitive long accessor. */ + public static class LongAccessor extends UnsafeAccessor { + public LongAccessor(Field field) { + super(field); + Preconditions.checkArgument(field.getType() == long.class); + } + + @Override + public Long get(Object obj) { + return getLong(obj); + } + + @Override + public long getLong(Object obj) { + checkObj(obj); + return UNSAFE.getLong(obj, fieldOffset); + } + + @Override + public void set(Object obj, Object value) { + putLong(obj, (Long) value); + } + + @Override + public void putLong(Object obj, long value) { + checkObj(obj); + UNSAFE.putLong(obj, fieldOffset, value); + } + } + + /** Primitive float accessor. */ + public static class FloatAccessor extends UnsafeAccessor { + public FloatAccessor(Field field) { + super(field); + Preconditions.checkArgument(field.getType() == float.class); + } + + @Override + public Object get(Object obj) { + return getFloat(obj); + } + + @Override + public float getFloat(Object obj) { + checkObj(obj); + return UNSAFE.getFloat(obj, fieldOffset); + } + + @Override + public void set(Object obj, Object value) { + putFloat(obj, (Float) value); + } + + @Override + public void putFloat(Object obj, float value) { + checkObj(obj); + UNSAFE.putFloat(obj, fieldOffset, value); + } + } + + /** Primitive double accessor. */ + public static class DoubleAccessor extends UnsafeAccessor { + public DoubleAccessor(Field field) { + super(field); + Preconditions.checkArgument(field.getType() == double.class); + } + + @Override + public Object get(Object obj) { + return getDouble(obj); + } + + @Override + public double getDouble(Object obj) { + checkObj(obj); + return UNSAFE.getDouble(obj, fieldOffset); + } + + @Override + public void set(Object obj, Object value) { + putDouble(obj, (Double) value); + } + + @Override + public void putDouble(Object obj, double value) { + checkObj(obj); + UNSAFE.putDouble(obj, fieldOffset, value); + } + } + + /** Object accessor. */ + public static class ObjectAccessor extends UnsafeAccessor { + public ObjectAccessor(Field field) { + super(field); + Preconditions.checkArgument(!TypeUtils.isPrimitive(field.getType())); + } + + @Override + public Object get(Object obj) { + checkObj(obj); + return UNSAFE.getObject(obj, fieldOffset); + } + + @Override + public void set(Object obj, Object value) { + checkObj(obj); + UNSAFE.putObject(obj, fieldOffset, value); + } + } + + static final class StaticObjectAccessor extends FieldAccessor { + private final Object base; + private final long offset; + + StaticObjectAccessor(Field field) { + super(field); + Preconditions.checkArgument(!TypeUtils.isPrimitive(field.getType())); + base = UNSAFE.staticFieldBase(field); + offset = UNSAFE.staticFieldOffset(field); + } + + @Override + public Object get(Object obj) { + return UNSAFE.getObject(base, offset); + } + + @Override + public void set(Object obj, Object value) { + UNSAFE.putObject(base, offset, value); + } + } + + static final class GeneratedAccessor extends FieldAccessor { + private static final ClassValueCache>> + cache = ClassValueCache.newClassKeyCache(8); + + private final MethodHandle getter; + private final MethodHandle setter; + + GeneratedAccessor(Field field) { + super(field); + ConcurrentMap> map = + cache.get(field.getDeclaringClass(), ConcurrentHashMap::new); + MethodHandles.Lookup lookup = _JDKAccess._trustedLookup(field.getDeclaringClass()); + Tuple2 tuple2 = + map.computeIfAbsent( + field.getName(), + k -> { + try { + MethodHandle getter = + lookup.findGetter( + field.getDeclaringClass(), field.getName(), field.getType()); + MethodHandle setter = + lookup.findSetter( + field.getDeclaringClass(), field.getName(), field.getType()); + return Tuple2.of(getter, setter); + } catch (IllegalAccessException | NoSuchFieldException ex) { + throw new RuntimeException(ex); + } + }); + getter = tuple2.f0; + setter = tuple2.f1; + } + + @Override + public Object get(Object obj) { + try { + return getter.invoke(obj); + } catch (Throwable e) { + throw new RuntimeException(e); + } + } + + @Override + public void set(Object obj, Object value) { + try { + setter.invoke(obj, value); + } catch (Throwable e) { + throw new RuntimeException(e); + } + } + + @Override + public boolean getBoolean(Object obj) { + try { + return (boolean) getter.invoke(obj); + } catch (Throwable e) { + throw new RuntimeException(e); + } + } + + @Override + public void putBoolean(Object obj, boolean value) { + try { + setter.invoke(obj, value); + } catch (Throwable e) { + throw new RuntimeException(e); + } + } + + @Override + public byte getByte(Object obj) { + try { + return (byte) getter.invoke(obj); + } catch (Throwable e) { + throw new RuntimeException(e); + } + } + + @Override + public void putByte(Object obj, byte value) { + try { + setter.invoke(obj, value); + } catch (Throwable e) { + throw new RuntimeException(e); + } + } + + @Override + public char getChar(Object obj) { + try { + return (char) getter.invoke(obj); + } catch (Throwable e) { + throw new RuntimeException(e); + } + } + + @Override + public void putChar(Object obj, char value) { + try { + setter.invoke(obj, value); + } catch (Throwable e) { + throw new RuntimeException(e); + } + } + + @Override + public short getShort(Object obj) { + try { + return (short) getter.invoke(obj); + } catch (Throwable e) { + throw new RuntimeException(e); + } + } + + @Override + public void putShort(Object obj, short value) { + try { + setter.invoke(obj, value); + } catch (Throwable e) { + throw new RuntimeException(e); + } + } + + @Override + public int getInt(Object obj) { + try { + return (int) getter.invoke(obj); + } catch (Throwable e) { + throw new RuntimeException(e); + } + } + + @Override + public void putInt(Object obj, int value) { + try { + setter.invoke(obj, value); + } catch (Throwable e) { + throw new RuntimeException(e); + } + } + + @Override + public long getLong(Object obj) { + try { + return (long) getter.invoke(obj); + } catch (Throwable e) { + throw new RuntimeException(e); + } + } + + @Override + public void putLong(Object obj, long value) { + try { + setter.invoke(obj, value); + } catch (Throwable e) { + throw new RuntimeException(e); + } + } + + @Override + public float getFloat(Object obj) { + try { + return (float) getter.invoke(obj); + } catch (Throwable e) { + throw new RuntimeException(e); + } + } + + @Override + public void putFloat(Object obj, float value) { + try { + setter.invoke(obj, value); + } catch (Throwable e) { + throw new RuntimeException(e); + } + } + + @Override + public double getDouble(Object obj) { + try { + return (double) getter.invoke(obj); + } catch (Throwable e) { + throw new RuntimeException(e); + } + } + + @Override + public void putDouble(Object obj, double value) { + try { + setter.invoke(obj, value); + } catch (Throwable e) { + throw new RuntimeException(e); + } + } + } +} diff --git a/java/fory-core/src/main/java/org/apache/fory/reflect/ObjectCreatorRegistry.java b/java/fory-core/src/main/java/org/apache/fory/reflect/ObjectCreatorRegistry.java new file mode 100644 index 0000000000..9932484af5 --- /dev/null +++ b/java/fory-core/src/main/java/org/apache/fory/reflect/ObjectCreatorRegistry.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.reflect; + +import java.lang.reflect.Constructor; +import org.apache.fory.annotation.Internal; +import org.apache.fory.collection.ClassValueCache; + +/** Runtime-scoped object creator cache and explicit constructor registrations. */ +@Internal +public final class ObjectCreatorRegistry { + private final ClassValueCache> objectCreatorCache = + ClassValueCache.newClassKeySoftCache(8); + private final ClassValueCache> constructorMatches = + ClassValueCache.newClassKeyCache(8); + + public ObjectCreator getObjectCreator(Class type) { + return (ObjectCreator) + objectCreatorCache.get( + type, + () -> + ObjectCreators.createObjectCreator( + type, + (ObjectCreators.ConstructorMatch) constructorMatches.getIfPresent(type))); + } + + public void registerConstructor( + Class type, Constructor constructor, String... fieldNames) { + ObjectCreators.ConstructorMatch match = + ObjectCreators.explicitConstructor(type, constructor, fieldNames.clone(), "registered"); + constructorMatches.put(type, match); + objectCreatorCache.put(type, ObjectCreators.createObjectCreator(type, match)); + } +} diff --git a/java/fory-core/src/main/java/org/apache/fory/reflect/ObjectCreators.java b/java/fory-core/src/main/java/org/apache/fory/reflect/ObjectCreators.java index d365c4e578..439525dcd9 100644 --- a/java/fory-core/src/main/java/org/apache/fory/reflect/ObjectCreators.java +++ b/java/fory-core/src/main/java/org/apache/fory/reflect/ObjectCreators.java @@ -19,21 +19,21 @@ package org.apache.fory.reflect; -import java.lang.annotation.Annotation; import java.lang.invoke.MethodHandle; import java.lang.invoke.MethodHandles.Lookup; import java.lang.invoke.MethodType; import java.lang.reflect.Constructor; import java.lang.reflect.Field; +import java.lang.reflect.Method; import java.lang.reflect.Modifier; -import java.lang.reflect.Parameter; import java.util.ArrayList; +import java.util.Arrays; import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Set; -import org.apache.fory.annotation.ForyField; +import org.apache.fory.annotation.ForyConstructor; import org.apache.fory.collection.ClassValueCache; import org.apache.fory.collection.Tuple2; import org.apache.fory.exception.ForyException; @@ -41,10 +41,10 @@ import org.apache.fory.platform.GraalvmSupport; import org.apache.fory.platform.JdkVersion; import org.apache.fory.platform.internal._JDKAccess; +import org.apache.fory.resolver.TypeResolver; import org.apache.fory.type.Descriptor; import org.apache.fory.type.TypeUtils; import org.apache.fory.util.record.RecordUtils; -import sun.misc.Unsafe; /** * Factory class for creating and caching {@link ObjectCreator} instances. @@ -60,8 +60,6 @@ * DeclaredNoArgCtrObjectCreator} with MethodHandle for fast invocation *

  • Classes without accessible constructors: Uses a private * constructor-bypassing creator on runtimes where that is still supported - *
  • GraalVM native image compatibility: Uses {@link - * ParentNoArgCtrObjectCreator} for constructor generate-based creation when needed *
  • Android compatibility: Uses reflection for records and no-arg * constructors, and throws when no supported reflective construction path exists * @@ -74,7 +72,6 @@ */ @SuppressWarnings("unchecked") public class ObjectCreators { - private static final Unsafe UNSAFE = AndroidSupport.IS_ANDROID ? null : _JDKAccess.UNSAFE; private static final ClassValueCache> cache = ClassValueCache.newClassKeySoftCache(8); @@ -92,25 +89,22 @@ public class ObjectCreators { * GraalVM native image) */ public static ObjectCreator getObjectCreator(Class type) { - return (ObjectCreator) cache.get(type, () -> creategetObjectCreator(type)); + return (ObjectCreator) cache.get(type, () -> createObjectCreator(type, null)); } - private static T allocateInstance(Class type) { - if (UNSAFE == null || JdkVersion.MAJOR_VERSION >= 25) { - throw new ForyException( - "Constructor-bypassing allocation is unsupported in this runtime for " + type); - } - try { - return (T) UNSAFE.allocateInstance(type); - } catch (InstantiationException e) { - throw new ForyException("Failed to allocate instance for " + type, e); - } + public static ObjectCreator getObjectCreator(TypeResolver typeResolver, Class type) { + return typeResolver.getSharedRegistry().getObjectCreatorRegistry().getObjectCreator(type); } - private static ObjectCreator creategetObjectCreator(Class type) { + static ObjectCreator createObjectCreator( + Class type, ConstructorMatch registeredConstructor) { if (RecordUtils.isRecord(type)) { return new RecordObjectCreator<>(type); } + ConstructorMatch explicitConstructor = explicitConstructor(type, registeredConstructor); + if (explicitConstructor != null) { + return new ConstructorObjectCreator<>(type, explicitConstructor); + } Constructor noArgConstructor = ReflectionUtils.getNoArgConstructor(type); if (AndroidSupport.IS_ANDROID) { if (noArgConstructor != null) { @@ -119,9 +113,6 @@ private static ObjectCreator creategetObjectCreator(Class type) { return new UnsupportedObjectCreator<>( type, "Android cannot create " + type + " without an accessible no-arg constructor"); } - if (JdkVersion.MAJOR_VERSION >= 25 && noArgConstructor == null) { - return new ConstructorObjectCreator<>(type); - } if (GraalvmSupport.IN_GRAALVM_NATIVE_IMAGE) { if (noArgConstructor != null) { return new DeclaredNoArgCtrObjectCreator<>(type); @@ -135,109 +126,121 @@ private static ObjectCreator creategetObjectCreator(Class type) { return new DeclaredNoArgCtrObjectCreator<>(type); } - public static boolean supportsJdk25Creation(Class type) { - if (JdkVersion.MAJOR_VERSION < 25 || RecordUtils.isRecord(type)) { - return true; - } - try { - ObjectCreator creator = creategetObjectCreator(type); - return !(creator instanceof UnsupportedObjectCreator); - } catch (RuntimeException e) { - return false; - } - } - - private static final class ConstructorMatch { + static final class ConstructorMatch { private final Constructor constructor; private final String[] fieldNames; private final Class[] declaringClasses; private final Class[] fieldTypes; private final boolean[] finalFields; - private final int score; private ConstructorMatch( Constructor constructor, String[] fieldNames, Class[] declaringClasses, Class[] fieldTypes, - boolean[] finalFields, - int score) { + boolean[] finalFields) { this.constructor = constructor; this.fieldNames = fieldNames; this.declaringClasses = declaringClasses; this.fieldTypes = fieldTypes; this.finalFields = finalFields; - this.score = score; } } - private static ConstructorMatch findConstructor(Class type) { + private static ConstructorMatch explicitConstructor( + Class type, ConstructorMatch registeredConstructor) { + if (registeredConstructor != null) { + return registeredConstructor; + } + Constructor annotatedConstructor = null; + ForyConstructor annotation = null; + for (Constructor constructor : type.getDeclaredConstructors()) { + ForyConstructor currentAnnotation = constructor.getAnnotation(ForyConstructor.class); + if (currentAnnotation == null) { + continue; + } + if (annotatedConstructor != null) { + throw new ForyException(type + " must not declare more than one @ForyConstructor"); + } + annotatedConstructor = constructor; + annotation = currentAnnotation; + } + if (annotatedConstructor == null) { + return null; + } + return explicitConstructor( + type, (Constructor) annotatedConstructor, annotation.value(), "@ForyConstructor"); + } + + static ConstructorMatch explicitConstructor( + Class type, Constructor constructor, String[] fieldNames, String source) { + if (constructor.getDeclaringClass() != type) { + throw new ForyException( + source + " constructor " + constructor + " does not belong to " + type); + } + if (fieldNames.length != constructor.getParameterCount()) { + throw new ForyException( + source + + " constructor " + + constructor + + " maps " + + fieldNames.length + + " fields to " + + constructor.getParameterCount() + + " parameters"); + } + ConstructorMatch match = + matchConstructorFields( + constructor, fieldsByExplicitNames(type, constructor, fieldNames, source)); + if (match == null) { + throw new ForyException( + source + + " constructor " + + constructor + + " parameter types do not match fields " + + Arrays.toString(fieldNames)); + } + return match; + } + + private static Field[] fieldsByExplicitNames( + Class type, Constructor constructor, String[] names, String source) { List fields = new ArrayList<>(); fields.addAll(Descriptor.getFields(type)); Map fieldsByName = new LinkedHashMap<>(); - Map> fieldsByNameList = new LinkedHashMap<>(); - Map fieldsById = new LinkedHashMap<>(); Set duplicateNames = new LinkedHashSet<>(); - Set duplicateIds = new LinkedHashSet<>(); for (Field field : fields) { - fieldsByNameList.computeIfAbsent(field.getName(), name -> new ArrayList<>()).add(field); Field previous = fieldsByName.put(field.getName(), field); if (previous != null) { duplicateNames.add(field.getName()); } - ForyField foryField = field.getAnnotation(ForyField.class); - if (foryField != null && foryField.id() >= 0) { - previous = fieldsById.put(foryField.id(), field); - if (previous != null) { - duplicateIds.add(foryField.id()); - } - } } - ConstructorMatch best = null; - for (Constructor constructor : type.getDeclaredConstructors()) { - if (constructor.isSynthetic()) { - continue; + Field[] constructorFields = new Field[names.length]; + Set seenNames = new LinkedHashSet<>(); + for (int i = 0; i < names.length; i++) { + String name = names[i]; + if (!seenNames.add(name)) { + throw new ForyException( + source + " constructor " + constructor + " maps " + name + " twice"); } - ConstructorMatch match = - matchConstructor( - type, - (Constructor) constructor, - fieldsByName, - fieldsByNameList, - fieldsById, - duplicateNames, - duplicateIds); - if (match != null && (best == null || match.score > best.score)) { - best = match; + if (duplicateNames.contains(name)) { + throw new ForyException( + source + + " constructor " + + constructor + + " cannot bind duplicate field name " + + name + + " in " + + type); } + Field field = fieldsByName.get(name); + if (field == null) { + throw new ForyException( + source + " constructor " + constructor + " maps unknown field " + name); + } + constructorFields[i] = field; } - if (best == null) { - throw new ForyException( - "JDK25 zero-Unsafe mode requires " - + "a bindable constructor because no no-arg constructor is available" - + " for " - + type - + ". Annotate the constructor with java.beans.ConstructorProperties or compile " - + "the class with -parameters."); - } - return best; - } - - private static ConstructorMatch matchConstructor( - Class type, - Constructor constructor, - Map fieldsByName, - Map> fieldsByNameList, - Map fieldsById, - Set duplicateNames, - Set duplicateIds) { - Field[] fields = - constructorFields( - constructor, fieldsByName, fieldsByNameList, fieldsById, duplicateNames, duplicateIds); - if (fields == null) { - return null; - } - return matchConstructorFields(constructor, fields); + return constructorFields; } private static ConstructorMatch matchConstructorFields( @@ -258,103 +261,7 @@ private static ConstructorMatch matchConstructorFields( finalFieldFlags[i] = Modifier.isFinal(field.getModifiers()); } return new ConstructorMatch<>( - constructor, names, declaringClasses, fieldTypes, finalFieldFlags, 300 - fields.length); - } - - private static Field[] constructorFields( - Constructor constructor, - Map fieldsByName, - Map> fieldsByNameList, - Map fieldsById, - Set duplicateNames, - Set duplicateIds) { - Field[] fields = fieldsByForyFieldId(constructor, fieldsById, duplicateIds); - if (fields != null) { - return fields; - } - String[] names = constructorFieldNames(constructor); - if (names != null) { - if (names.length != constructor.getParameterCount()) { - return null; - } - return fieldsByName(constructor, fieldsByName, fieldsByNameList, duplicateNames, names); - } - return null; - } - - private static Field[] fieldsByForyFieldId( - Constructor constructor, Map fieldsById, Set duplicateIds) { - Parameter[] parameters = constructor.getParameters(); - Field[] fields = new Field[parameters.length]; - boolean hasForyFieldId = false; - for (int i = 0; i < parameters.length; i++) { - ForyField foryField = parameters[i].getAnnotation(ForyField.class); - if (foryField == null || foryField.id() < 0) { - continue; - } - hasForyFieldId = true; - int id = foryField.id(); - if (duplicateIds.contains(id)) { - throw new ForyException("Constructor parameter id " + id + " is ambiguous"); - } - fields[i] = fieldsById.get(id); - if (fields[i] == null) { - return null; - } - } - if (!hasForyFieldId) { - return null; - } - for (Field field : fields) { - if (field == null) { - return null; - } - } - return fields; - } - - private static Field[] fieldsByName( - Constructor constructor, - Map fieldsByName, - Map> fieldsByNameList, - Set duplicateNames, - String[] names) { - Field[] fields = new Field[names.length]; - for (int i = 0; i < names.length; i++) { - String name = names[i]; - if (duplicateNames.contains(name)) { - Field field = syntheticField(constructor, fieldsByNameList.get(name)); - if (field == null) { - throw new ForyException( - "Constructor parameter " - + name - + " is ambiguous because " - + constructor.getDeclaringClass() - + " has duplicate field names"); - } - fields[i] = field; - continue; - } - Field field = fieldsByName.get(name); - if (field == null) { - return null; - } - fields[i] = field; - } - return fields; - } - - private static Field syntheticField(Constructor constructor, List fields) { - if (fields == null) { - return null; - } - Class declaringClass = constructor.getDeclaringClass(); - for (Field field : fields) { - if (field.isSynthetic() && field.getDeclaringClass() == declaringClass) { - return field; - } - } - return null; + constructor, names, declaringClasses, fieldTypes, finalFieldFlags); } private static boolean constructorTypeMatches(Class parameterType, Field field) { @@ -363,45 +270,16 @@ private static boolean constructorTypeMatches(Class parameterType, Field fiel return boxedParameterType.isAssignableFrom(boxedFieldType); } - private static String[] constructorFieldNames(Constructor constructor) { - String[] names = constructorProperties(constructor); - if (names != null) { - return names; - } - Parameter[] parameters = constructor.getParameters(); - for (Parameter parameter : parameters) { - if (!parameter.isNamePresent()) { - return null; - } - } - names = new String[parameters.length]; - for (int i = 0; i < parameters.length; i++) { - names[i] = parameters[i].getName(); - } - return names; - } - - private static String[] constructorProperties(Constructor constructor) { - for (Annotation annotation : constructor.getDeclaredAnnotations()) { - if ("java.beans.ConstructorProperties".equals(annotation.annotationType().getName())) { - try { - return (String[]) annotation.annotationType().getMethod("value").invoke(annotation); - } catch (ReflectiveOperationException e) { - throw new ForyException("Failed to read ConstructorProperties for " + constructor, e); - } - } - } - return null; - } - private static MethodHandle constructorHandle(Class type, Constructor constructor) { Lookup lookup = _JDKAccess._trustedLookup(type); if (lookup == null) { return null; } try { - return lookup.findConstructor( - type, MethodType.methodType(void.class, constructor.getParameterTypes())); + MethodHandle handle = + lookup.findConstructor( + type, MethodType.methodType(void.class, constructor.getParameterTypes())); + return handle.asSpreader(Object[].class, constructor.getParameterCount()); } catch (NoSuchMethodException | IllegalAccessException e) { return null; } @@ -462,9 +340,8 @@ public static final class ConstructorObjectCreator extends ObjectCreator { private final Class[] fieldTypes; private final boolean[] finalFields; - private ConstructorObjectCreator(Class type) { + private ConstructorObjectCreator(Class type, ConstructorMatch match) { super(type); - ConstructorMatch match = findConstructor(type); constructor = match.constructor; handle = constructorHandle(type, constructor); fieldNames = match.fieldNames; @@ -485,22 +362,22 @@ public boolean hasConstructorFields() { @Override public String[] getConstructorFieldNames() { - return fieldNames; + return fieldNames.clone(); } @Override public Class[] getConstructorFieldDeclaringClasses() { - return declaringClasses; + return declaringClasses.clone(); } @Override public Class[] getConstructorFieldTypes() { - return fieldTypes; + return fieldTypes.clone(); } @Override public boolean[] getConstructorFieldFinal() { - return finalFields; + return finalFields.clone(); } @Override @@ -535,7 +412,7 @@ public T newInstanceWithArguments(Object... arguments) { if (handle == null) { return constructor.newInstance(arguments); } - return (T) handle.invokeWithArguments(arguments); + return (T) handle.invoke(arguments); } catch (Throwable e) { throw new ForyException("Failed to create instance using constructor: " + type, e); } @@ -550,7 +427,7 @@ public UnsafeObjectCreator(Class type) { @Override public T newInstance() { - return ObjectCreators.allocateInstance(type); + return UnsafeObjectAllocator.allocate(type); } @Override @@ -590,7 +467,10 @@ public RecordObjectCreator(Class type) { super(type); Tuple2 tuple2 = RecordUtils.getRecordConstructor(type); constructor = tuple2.f0; - handle = tuple2.f1; + handle = + tuple2.f1 == null + ? null + : tuple2.f1.asSpreader(Object[].class, constructor.getParameterCount()); if (AndroidSupport.IS_ANDROID || (GraalvmSupport.IN_GRAALVM_NATIVE_IMAGE && JdkVersion.MAJOR_VERSION >= 25)) { try { @@ -617,7 +497,7 @@ public T newInstanceWithArguments(Object... arguments) { return (T) constructor.newInstance(arguments); } else { // Regular path: use method handle - return (T) handle.invokeWithArguments(arguments); + return (T) handle.invoke(arguments); } } catch (Throwable e) { throw new ForyException("Failed to create record instance: " + type, e); @@ -627,42 +507,32 @@ public T newInstanceWithArguments(Object... arguments) { public static final class ParentNoArgCtrObjectCreator extends ObjectCreator { private static volatile Object reflectionFactory; - private static volatile MethodHandle newConstructorForSerializationMethod; + private static volatile Method newConstructorForSerializationMethod; private final Constructor constructor; public ParentNoArgCtrObjectCreator(Class type) { super(type); + if (JdkVersion.MAJOR_VERSION >= 25) { + throw new ForyException( + "ReflectionFactory object creation is unavailable in JDK25+ zero-Unsafe mode for " + + type); + } this.constructor = createSerializationConstructor(type); } private static Constructor createSerializationConstructor(Class type) { try { - // Get ReflectionFactory instance if (reflectionFactory == null) { - Class reflectionFactoryClass; - if (JdkVersion.MAJOR_VERSION >= 9) { - reflectionFactoryClass = Class.forName("jdk.internal.reflect.ReflectionFactory"); - } else { - reflectionFactoryClass = Class.forName("sun.reflect.ReflectionFactory"); - } - Lookup lookup = _JDKAccess._trustedLookup(reflectionFactoryClass); - MethodHandle handle = - lookup.findStatic( - reflectionFactoryClass, - "getReflectionFactory", - MethodType.methodType(reflectionFactoryClass)); - reflectionFactory = handle.invoke(); + Class reflectionFactoryClass = Class.forName("sun.reflect.ReflectionFactory"); + Method getReflectionFactory = reflectionFactoryClass.getMethod("getReflectionFactory"); + reflectionFactory = getReflectionFactory.invoke(null); newConstructorForSerializationMethod = - lookup.findVirtual( - reflectionFactoryClass, - "newConstructorForSerialization", - MethodType.methodType(Constructor.class, Class.class, Constructor.class)); + reflectionFactoryClass.getMethod( + "newConstructorForSerialization", Class.class, Constructor.class); } - // Find a public no-arg constructor in parent classes that we can use as a template Constructor parentConstructor = findPublicNoArgConstructor(type); if (parentConstructor == null) { - // Use Object's constructor as fallback parentConstructor = Object.class.getDeclaredConstructor(); } else { try { @@ -671,7 +541,6 @@ private static Constructor createSerializationConstructor(Class type) parentConstructor = Object.class.getDeclaredConstructor(); } } - // Create serialization constructor using ReflectionFactory return (Constructor) newConstructorForSerializationMethod.invoke(reflectionFactory, type, parentConstructor); } catch (Throwable e) { @@ -685,7 +554,7 @@ private static Constructor findPublicNoArgConstructor(Class type) { while (current != null && current != Object.class) { try { Constructor constructor = current.getDeclaredConstructor(); - if (constructor.getModifiers() == java.lang.reflect.Modifier.PUBLIC) { + if (Modifier.isPublic(constructor.getModifiers())) { return constructor; } } catch (NoSuchMethodException ignored) { diff --git a/java/fory-core/src/main/java/org/apache/fory/reflect/ReflectionFieldAccessor.java b/java/fory-core/src/main/java/org/apache/fory/reflect/ReflectionFieldAccessor.java index 3ac32c4625..82ffe00e47 100644 --- a/java/fory-core/src/main/java/org/apache/fory/reflect/ReflectionFieldAccessor.java +++ b/java/fory-core/src/main/java/org/apache/fory/reflect/ReflectionFieldAccessor.java @@ -24,7 +24,7 @@ final class ReflectionFieldAccessor extends FieldAccessor { ReflectionFieldAccessor(Field field) { - super(field, -1); + super(field); try { field.setAccessible(true); } catch (RuntimeException e) { diff --git a/java/fory-core/src/main/java/org/apache/fory/reflect/UnsafeObjectAllocator.java b/java/fory-core/src/main/java/org/apache/fory/reflect/UnsafeObjectAllocator.java new file mode 100644 index 0000000000..0e9d1b0dd3 --- /dev/null +++ b/java/fory-core/src/main/java/org/apache/fory/reflect/UnsafeObjectAllocator.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.reflect; + +import org.apache.fory.annotation.Internal; +import org.apache.fory.exception.ForyException; +import org.apache.fory.platform.AndroidSupport; +import org.apache.fory.platform.JdkVersion; +import org.apache.fory.platform.internal._JDKAccess; +import sun.misc.Unsafe; + +/** Internal JDK8-24 allocator used by object creators. */ +@Internal +final class UnsafeObjectAllocator { + private static final Unsafe UNSAFE = AndroidSupport.IS_ANDROID ? null : _JDKAccess.UNSAFE; + + private UnsafeObjectAllocator() {} + + static T allocate(Class type) { + if (UNSAFE == null || JdkVersion.MAJOR_VERSION >= 25) { + throw new ForyException( + "Constructor-bypassing Unsafe allocation is unsupported in this runtime for " + type); + } + try { + return (T) UNSAFE.allocateInstance(type); + } catch (InstantiationException e) { + throw new ForyException("Failed to allocate instance for " + type, e); + } + } +} diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java index 787505191e..48404bccde 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java @@ -32,7 +32,6 @@ import java.io.Serializable; import java.lang.invoke.SerializedLambda; import java.lang.reflect.Method; -import java.lang.reflect.Modifier; import java.math.BigDecimal; import java.math.BigInteger; import java.net.URL; @@ -116,8 +115,6 @@ import org.apache.fory.meta.TypeExtMeta; import org.apache.fory.platform.AndroidSupport; import org.apache.fory.platform.GraalvmSupport; -import org.apache.fory.platform.JdkVersion; -import org.apache.fory.reflect.ObjectCreators; import org.apache.fory.reflect.ReflectionUtils; import org.apache.fory.serializer.ArraySerializers; import org.apache.fory.serializer.BufferSerializers; @@ -1484,7 +1481,7 @@ public Class getSerializerClass(Class cls, boolean code if (!MemoryUtils.JDK_COLLECTION_FIELD_ACCESS) { throw new UnsupportedOperationException( "StringTokenizer serialization requires JDK internal field access. On JDK25+, open " - + "java.base/java.util to org.apache.fory.core,org.apache.fory.format."); + + "java.base/java.lang.invoke to org.apache.fory.core."); } return Serializers.StringTokenizerSerializer.class; } else if (ByteBuffer.class.isAssignableFrom(cls)) { @@ -1546,9 +1543,6 @@ public Class getSerializerClass(Class cls, boolean code return MapSerializer.class; } } - if (requiresJdkStream(cls)) { - return getDefaultJDKStreamSerializerType(); - } if (isCrossLanguage()) { LOG.warn("Class {} isn't supported for cross-language serialization.", cls); } @@ -1587,9 +1581,6 @@ public Object id() { public Class getObjectSerializerClass( Class cls, JITContext.SerializerJITCallback> callback) { boolean codegen = config.isCodeGenEnabled() && supportCodegenForJavaSerialization(cls); - if (JdkVersion.MAJOR_VERSION >= 25 && !Modifier.isPublic(cls.getModifiers())) { - codegen = false; - } return getObjectSerializerClass(cls, false, codegen, callback); } @@ -1639,22 +1630,6 @@ public Class getObjectSerializerClass( } } - private static boolean requiresJdkStream(Class cls) { - return JdkVersion.MAJOR_VERSION >= 25 - && cls.getName().startsWith("java.") - && Serializable.class.isAssignableFrom(cls) - && !hasNoArgConstructor(cls); - } - - private static boolean hasNoArgConstructor(Class cls) { - try { - cls.getDeclaredConstructor(); - return true; - } catch (NoSuchMethodException e) { - return false; - } - } - public Class getJavaSerializer(Class clz) { if (Collection.class.isAssignableFrom(clz)) { return CollectionSerializers.JDKCompatibleCollectionSerializer.class; @@ -1906,7 +1881,7 @@ private void registerGraalvmSerializerClass(Class cls) { RecordUtils.getRecordComponents(cls); } if (needsGraalvmObjectCreator(cls, serializerClass)) { - ObjectCreators.getObjectCreator(cls); + getObjectCreator(cls); } } diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/SharedRegistry.java b/java/fory-core/src/main/java/org/apache/fory/resolver/SharedRegistry.java index 2eb57fd705..58863c88d9 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/SharedRegistry.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/SharedRegistry.java @@ -39,6 +39,7 @@ import org.apache.fory.meta.MetaStringEncoder; import org.apache.fory.meta.TypeDef; import org.apache.fory.platform.GraalvmSupport; +import org.apache.fory.reflect.ObjectCreatorRegistry; import org.apache.fory.serializer.Serializer; import org.apache.fory.type.Descriptor; import org.apache.fory.type.DescriptorGrouper; @@ -81,6 +82,7 @@ public final class SharedRegistry { new ConcurrentIdentityMap<>(); final ConcurrentIdentityMap, Serializer> registeredSerializerCache = new ConcurrentIdentityMap<>(); + private final ObjectCreatorRegistry objectCreatorRegistry = new ObjectCreatorRegistry(); final StaticGeneratedSerializerRegistry staticGeneratedSerializerRegistry = new StaticGeneratedSerializerRegistry(); private final Object metaStringCacheLock = new Object(); @@ -125,6 +127,10 @@ Serializer cacheRegisteredSerializer(Class type, Serializer serializer) return existing; } + public ObjectCreatorRegistry getObjectCreatorRegistry() { + return objectCreatorRegistry; + } + TypeInfo cacheRegisteredTypeInfo(Class type, TypeInfo typeInfo) { TypeInfo existing = registeredTypeInfoCache.putIfAbsent(type, typeInfo); return existing == null ? typeInfo : existing; diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java index d60a72ce3c..3288370cd2 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java @@ -77,6 +77,7 @@ import org.apache.fory.meta.TypeExtMeta; import org.apache.fory.platform.AndroidSupport; import org.apache.fory.platform.GraalvmSupport; +import org.apache.fory.reflect.ObjectCreator; import org.apache.fory.reflect.ReflectionUtils; import org.apache.fory.reflect.TypeRef; import org.apache.fory.serializer.CodegenSerializer; @@ -341,6 +342,17 @@ public abstract void registerUnion( public abstract void registerEnum( Class type, String namespace, String typeName, Serializer serializer); + /** + * Returns the runtime-scoped object creator for {@code type}. + * + *

    The creator respects constructor mappings registered through {@code Fory} and annotations on + * the target type. + */ + @Internal + public final ObjectCreator getObjectCreator(Class type) { + return sharedRegistry.getObjectCreatorRegistry().getObjectCreator(type); + } + /** * Registers a custom serializer for a type. * diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/AbstractObjectSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/AbstractObjectSerializer.java index 8b8dc6376b..05460d0b56 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/AbstractObjectSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/AbstractObjectSerializer.java @@ -26,6 +26,7 @@ import java.util.List; import java.util.stream.Collectors; import org.apache.fory.Fory; +import org.apache.fory.collection.IntArray; import org.apache.fory.config.Config; import org.apache.fory.context.CopyContext; import org.apache.fory.context.ReadContext; @@ -38,7 +39,6 @@ import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.reflect.FieldAccessor; import org.apache.fory.reflect.ObjectCreator; -import org.apache.fory.reflect.ObjectCreators; import org.apache.fory.reflect.ReflectionUtils; import org.apache.fory.reflect.TypeRef; import org.apache.fory.resolver.RefMode; @@ -64,6 +64,11 @@ public abstract class AbstractObjectSerializer extends Serializer { private static final Logger LOG = LoggerFactory.getLogger(AbstractObjectSerializer.class); private static final Object SELF_REFERENCE = new Object(); + // Constructor-bound objects reserve a ref id before constructor arguments are read, but the + // context package must stay a generic ref owner. Track unresolved constructor refs here so + // final-field construction does not add context APIs or bind the wrong pending ref id. + private static final Object CONSTRUCTOR_REF_IDS = new Object(); + private static final Object UNRESOLVED_CONSTRUCTOR_REF_IDS = new Object(); protected final Config config; protected final TypeResolver typeResolver; protected final boolean isRecord; @@ -80,7 +85,7 @@ protected AbstractObjectSerializer() { } public AbstractObjectSerializer(TypeResolver typeResolver, Class type) { - this(typeResolver, type, ObjectCreators.getObjectCreator(type)); + this(typeResolver, type, typeResolver.getObjectCreator(type)); } public AbstractObjectSerializer( @@ -161,6 +166,7 @@ static Object readField( MemoryBuffer buffer) { if (fieldInfo.useDeclaredTypeInfo) { if (refMode == RefMode.TRACKING) { + trackConstructorRefRead(readContext, buffer); return readContext.readRef(fieldInfo.typeInfo); } if (refMode != RefMode.NULL_ONLY || buffer.readByte() != Fory.NULL_FLAG) { @@ -170,6 +176,7 @@ static Object readField( return null; } if (refMode == RefMode.TRACKING) { + trackConstructorRefRead(readContext, buffer); int nextReadRefId = refReader.tryPreserveRefId(buffer); if (nextReadRefId >= Fory.NOT_NULL_VALUE_FLAG) { Object value = @@ -896,20 +903,27 @@ private T copyConstructorObject(CopyContext copyContext, T originObj) { int[] constructorFieldIndexes = buildConstructorFieldIndexes(fieldInfos); boolean[] constructorFieldMask = buildConstructorFieldMask(fieldInfos.length, constructorFieldIndexes); - copyContext.markCopying(originObj); - try { - Object[] fieldValues = - copyFieldValues(copyContext, originObj, fieldInfos, constructorFieldMask, true); - T newObj = - objectCreator.newInstanceWithArguments( - constructorArgs( - fieldValues, constructorFieldIndexes, objectCreator.getConstructorFieldTypes())); - copyContext.reference(originObj, newObj); - copyFields(copyContext, fieldInfos, originObj, newObj, constructorFieldMask, false); - return newObj; - } catch (Throwable e) { - copyContext.cancelCopy(originObj); - throw e; + checkNoSelfConstructorField(originObj, fieldInfos, constructorFieldMask); + Object[] fieldValues = + copyFieldValues(copyContext, originObj, fieldInfos, constructorFieldMask, true); + T newObj = + objectCreator.newInstanceWithArguments( + constructorArgs( + fieldValues, constructorFieldIndexes, objectCreator.getConstructorFieldTypes())); + copyContext.reference(originObj, newObj); + copyFields(copyContext, fieldInfos, originObj, newObj, constructorFieldMask, false); + return newObj; + } + + private void checkNoSelfConstructorField( + T originObj, SerializationFieldInfo[] fieldInfos, boolean[] constructorFieldMask) { + for (int i = 0; i < fieldInfos.length; i++) { + if (constructorFieldMask[i] && !fieldInfos[i].isPrimitiveField) { + Object fieldValue = fieldInfos[i].fieldAccessor.getObject(originObj); + if (fieldValue == originObj) { + throwConstructorCycle(type); + } + } } } @@ -1246,20 +1260,24 @@ public static void checkNoUnresolvedReadRef(ReadContext readContext, Class ty public static void beginConstructorRef(ReadContext readContext) { if (readContext.hasPreservedRefId()) { - readContext.trackUnresolvedRef(readContext.lastPreservedRefId()); + constructorRefIds(readContext).add(readContext.lastPreservedRefId()); } } public static void endConstructorRef(ReadContext readContext) { - readContext.untrackUnresolvedRef(); + IntArray refIds = (IntArray) readContext.getContextObject(CONSTRUCTOR_REF_IDS); + if (refIds != null && refIds.size > 0) { + refIds.pop(); + } } public static void referenceConstructorRef(ReadContext readContext, Object object) { - if (readContext.hasTrackedRef()) { - // Constructor-bound objects are registered after some fields may already have reserved and - // resolved their own ids, so bind the tracked id instead of popping the current stack top. - readContext.reference(readContext.currentTrackedRefId(), object); - } else if (readContext.hasPreservedRefId()) { + int constructorRefId = currentConstructorRefId(readContext); + if (constructorRefId >= 0) { + readContext.setReadRef(constructorRefId, object); + return; + } + if (readContext.hasPreservedRefId()) { readContext.reference(object); } } @@ -1287,14 +1305,87 @@ public static Object resolveBufferedValue(Object value, Object targetObject) { } private static boolean consumeSelfRef(ReadContext readContext) { - if (readContext.hasTrackedRef()) { - return readContext.consumeUnresolvedRef(readContext.currentTrackedRefId()); + int constructorRefId = currentConstructorRefId(readContext); + return constructorRefId >= 0 && consumeUnresolvedConstructorRef(readContext, constructorRefId); + } + + public static void trackConstructorRefRead(ReadContext readContext, MemoryBuffer buffer) { + IntArray constructorRefIds = (IntArray) readContext.getContextObject(CONSTRUCTOR_REF_IDS); + if (constructorRefIds == null || constructorRefIds.size == 0) { + return; + } + int readerIndex = buffer.readerIndex(); + if (buffer.getByte(readerIndex) != Fory.REF_FLAG) { + return; + } + int refId; + try { + buffer.readerIndex(readerIndex + 1); + refId = buffer.readVarUInt32Small14(); + } finally { + buffer.readerIndex(readerIndex); + } + if (readContext.getReadRef(refId) == null && containsRefId(constructorRefIds, refId)) { + unresolvedConstructorRefIds(readContext).add(refId); + } + } + + private static IntArray constructorRefIds(ReadContext readContext) { + IntArray refIds = (IntArray) readContext.getContextObject(CONSTRUCTOR_REF_IDS); + if (refIds == null) { + refIds = new IntArray(4); + readContext.putContextObject(CONSTRUCTOR_REF_IDS, refIds); + } + return refIds; + } + + private static IntArray unresolvedConstructorRefIds(ReadContext readContext) { + IntArray refIds = (IntArray) readContext.getContextObject(UNRESOLVED_CONSTRUCTOR_REF_IDS); + if (refIds == null) { + refIds = new IntArray(4); + readContext.putContextObject(UNRESOLVED_CONSTRUCTOR_REF_IDS, refIds); + } + return refIds; + } + + private static int currentConstructorRefId(ReadContext readContext) { + IntArray refIds = (IntArray) readContext.getContextObject(CONSTRUCTOR_REF_IDS); + if (refIds == null || refIds.size == 0) { + return -1; + } + return refIds.get(refIds.size - 1); + } + + private static boolean containsRefId(IntArray refIds, int refId) { + for (int i = refIds.size - 1; i >= 0; i--) { + if (refIds.get(i) == refId) { + return true; + } + } + return false; + } + + private static boolean consumeUnresolvedConstructorRef(ReadContext readContext, int refId) { + IntArray unresolvedRefIds = + (IntArray) readContext.getContextObject(UNRESOLVED_CONSTRUCTOR_REF_IDS); + if (unresolvedRefIds == null || unresolvedRefIds.size == 0) { + return false; + } + boolean found = false; + int newSize = 0; + for (int i = 0; i < unresolvedRefIds.size; i++) { + int unresolvedRefId = unresolvedRefIds.get(i); + if (unresolvedRefId == refId) { + found = true; + } else { + unresolvedRefIds.elementData[newSize++] = unresolvedRefId; + } } - return readContext.hasPreservedRefId() - && readContext.consumeUnresolvedRef(readContext.lastPreservedRefId()); + unresolvedRefIds.size = newSize; + return found; } - private static void throwConstructorCycle(Class type) { + protected static void throwConstructorCycle(Class type) { throw new ForyException( "Cyclic references to constructor-bound type " + type.getName() diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/CompatibleLayerSerializerBase.java b/java/fory-core/src/main/java/org/apache/fory/serializer/CompatibleLayerSerializerBase.java index 611dfc0da4..a93a2f250f 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/CompatibleLayerSerializerBase.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/CompatibleLayerSerializerBase.java @@ -21,6 +21,7 @@ import org.apache.fory.collection.IdentityObjectIntMap; import org.apache.fory.collection.ObjectIntMap; +import org.apache.fory.context.CopyContext; import org.apache.fory.context.MetaWriteContext; import org.apache.fory.context.ReadContext; import org.apache.fory.context.RefReader; @@ -199,6 +200,15 @@ public Object[] getFieldValuesForPutFields( return vals; } + @SuppressWarnings("rawtypes") + public Object[] copyFieldValuesForPutFields( + CopyContext copyContext, Object obj, ObjectIntMap fieldIndexMap, int arraySize) { + checkLayerSerializerMeta(); + Object[] vals = new Object[arraySize]; + collectCopiedPutFieldValues(copyContext, obj, fieldIndexMap, vals, allFields); + return vals; + } + public void skipFields(ReadContext readContext) { checkLayerSerializerMeta(); MemoryBuffer buffer = readContext.getBuffer(); @@ -361,4 +371,27 @@ private void collectPutFieldValues( } } } + + @SuppressWarnings("rawtypes") + private void collectCopiedPutFieldValues( + CopyContext copyContext, + Object obj, + ObjectIntMap fieldIndexMap, + Object[] vals, + SerializationFieldInfo[] fieldInfos) { + for (SerializationFieldInfo fieldInfo : fieldInfos) { + FieldAccessor fieldAccessor = fieldInfo.fieldAccessor; + if (fieldAccessor != null) { + String fieldName = fieldAccessor.getField().getName(); + int index = fieldIndexMap.get(fieldName, -1); + if (index != -1 && index < vals.length) { + Object fieldValue = fieldAccessor.get(obj); + vals[index] = + fieldInfo.isPrimitiveField || fieldValue == null + ? fieldValue + : copyContext.copyObject(fieldValue); + } + } + } + } } diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/CompatibleSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/CompatibleSerializer.java index 01058fbd81..44db7e6b86 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/CompatibleSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/CompatibleSerializer.java @@ -35,7 +35,6 @@ import org.apache.fory.platform.GraalvmSupport; import org.apache.fory.platform.JdkVersion; import org.apache.fory.reflect.FieldAccessor; -import org.apache.fory.reflect.ObjectCreators; import org.apache.fory.resolver.ClassResolver; import org.apache.fory.resolver.RefMode; import org.apache.fory.resolver.TypeResolver; @@ -267,7 +266,7 @@ private T newInstance() { || GraalvmSupport.IN_GRAALVM_NATIVE_IMAGE || JdkVersion.MAJOR_VERSION >= 25 ? newBean() - : ObjectCreators.getObjectCreator(type).newInstance(); + : typeResolver.getObjectCreator(type).newInstance(); // Set default values for missing fields in Scala case classes DefaultValueUtils.setDefaultValues(obj, defaultValueFields); return obj; diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/ExceptionSerializers.java b/java/fory-core/src/main/java/org/apache/fory/serializer/ExceptionSerializers.java index e4d3ec4b02..547b97c62c 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/ExceptionSerializers.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/ExceptionSerializers.java @@ -42,6 +42,7 @@ import org.apache.fory.meta.TypeDef; import org.apache.fory.platform.AndroidSupport; import org.apache.fory.platform.GraalvmSupport; +import org.apache.fory.platform.JdkVersion; import org.apache.fory.platform.internal._JDKAccess; import org.apache.fory.reflect.FieldAccessor; import org.apache.fory.reflect.ObjectCreator; @@ -88,7 +89,7 @@ public ExceptionSerializer(TypeResolver typeResolver, Class type) { messageConstructor = getOptionalMessageConstructor(type); objectCreator = messageConstructor == null && MemoryUtils.JDK_LANG_FIELD_ACCESS - ? createThrowableObjectCreator(type) + ? createThrowableObjectCreator(typeResolver, type) : null; slotsSerializers = buildSlotsSerializers(typeResolver, type); if (!MemoryUtils.JDK_LANG_FIELD_ACCESS @@ -98,7 +99,7 @@ && hasSubclassFields(slotsSerializers)) { "Throwable serialization for JDK type " + type.getName() + " with subclass fields requires JDK internal field access. On JDK25+, open " - + "java.base/java.lang to org.apache.fory.core,org.apache.fory.format."); + + "java.base/java.lang.invoke to org.apache.fory.core."); } // Native-image runtime must rebuild slot serializers once so field accessors and // descriptors are created against the runtime heap layout instead of reusing @@ -158,8 +159,7 @@ private T readAndroidThrowableWithoutDetailMessageField( "Deserializing Throwable type " + type.getName() + " without a String message constructor requires JDK internal field access. " - + "On JDK25+, open java.base/java.lang to " - + "org.apache.fory.core,org.apache.fory.format."); + + "On JDK25+, open java.base/java.lang.invoke to org.apache.fory.core."); } int refId = readContext.lastPreservedRefId(); if (refId >= 0) { @@ -173,8 +173,8 @@ private T readAndroidThrowableWithoutDetailMessageField( throw new ForyException( "Deserializing cyclic Throwable references for type " + type.getName() - + " requires JDK internal field access. On JDK25+, open java.base/java.lang " - + "to org.apache.fory.core,org.apache.fory.format."); + + " requires JDK internal field access. On JDK25+, open java.base/java.lang.invoke " + + "to org.apache.fory.core."); } T obj = newThrowableWithMessage(detailMessage); readContext.reference(obj); @@ -383,12 +383,12 @@ private static StackTraceElement newStackTraceElement( } private static ObjectCreator createThrowableObjectCreator( - Class type) { - if (GraalvmSupport.IN_GRAALVM_NATIVE_IMAGE) { - return ObjectCreators.getObjectCreator(type); + TypeResolver typeResolver, Class type) { + if (GraalvmSupport.IN_GRAALVM_NATIVE_IMAGE || JdkVersion.MAJOR_VERSION >= 25) { + return typeResolver.getObjectCreator(type); } if (ReflectionUtils.getCtrHandle(type, false) != null) { - return ObjectCreators.getObjectCreator(type); + return typeResolver.getObjectCreator(type); } return new ObjectCreators.ParentNoArgCtrObjectCreator<>(type); } diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectSerializer.java index af09950c50..eb84189eec 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectSerializer.java @@ -34,7 +34,6 @@ import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.meta.TypeDef; import org.apache.fory.reflect.ObjectCreator; -import org.apache.fory.reflect.ObjectCreators; import org.apache.fory.resolver.TypeResolver; import org.apache.fory.serializer.FieldGroups.SerializationFieldInfo; import org.apache.fory.serializer.struct.Fingerprint; @@ -74,7 +73,7 @@ public ObjectSerializer(TypeResolver typeResolver, Class cls) { } public ObjectSerializer(TypeResolver typeResolver, Class cls, boolean resolveParent) { - this(typeResolver, cls, resolveParent, ObjectCreators.getObjectCreator(cls)); + this(typeResolver, cls, resolveParent, typeResolver.getObjectCreator(cls)); } public ObjectSerializer( diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectStreamSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectStreamSerializer.java index 631541e5d9..6b14aad182 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectStreamSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectStreamSerializer.java @@ -31,6 +31,7 @@ import java.io.ObjectStreamField; import java.io.Serializable; import java.io.UnsupportedEncodingException; +import java.lang.invoke.MethodHandle; import java.lang.invoke.MethodHandles; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; @@ -39,7 +40,6 @@ import java.util.Collection; import java.util.Collections; import java.util.List; -import java.util.Map; import java.util.TreeMap; import java.util.function.BiConsumer; import java.util.function.Consumer; @@ -64,15 +64,12 @@ import org.apache.fory.meta.TypeDef; import org.apache.fory.platform.AndroidSupport; import org.apache.fory.platform.GraalvmSupport; -import org.apache.fory.platform.JdkVersion; import org.apache.fory.platform.internal._JDKAccess; import org.apache.fory.reflect.ObjectCreator; -import org.apache.fory.reflect.ObjectCreators; import org.apache.fory.resolver.ClassResolver; import org.apache.fory.resolver.TypeInfo; import org.apache.fory.resolver.TypeResolver; import org.apache.fory.serializer.PrimitiveSerializers.LongSerializer; -import org.apache.fory.serializer.collection.ChildContainerSerializers; import org.apache.fory.type.Descriptor; import org.apache.fory.type.TypeUtils; import org.apache.fory.type.Types; @@ -85,10 +82,12 @@ *

      *
    • `writeObject/readObject` occurs only at current class in class hierarchy. *
    • `writeReplace/readResolve` don't occur in class hierarchy. - *
    • class hierarchy doesn't have duplicated fields. If any of those conditions are not met, - * fallback jdk custom serialization to {@link JavaSerializer}. + *
    • class hierarchy doesn't have duplicated fields. *
    * + *

    Serializer selection must not change by JDK version. Types that use this serializer on JDK8 + * through JDK24 must keep this serializer on JDK25+ so existing Fory bytes remain readable. + * *

    `ObjectInputStream#setObjectInputFilter` will be ignored by this serializer. */ @SuppressWarnings({"unchecked", "rawtypes"}) @@ -97,7 +96,6 @@ public class ObjectStreamSerializer extends AbstractObjectSerializer { private static final int MAX_CACHED_TYPE_DEFS = 8192; private final SlotInfo[] slotsInfos; - private final Serializer fallbackSerializer; // Instance-level cache: TypeDef ID -> TypeInfo (shared across all slots). private final LongMap typeDefIdToTypeInfo = new LongMap<>(4, 0.4f); @@ -188,17 +186,11 @@ private static ObjectStreamClass safeObjectStreamClassLookup(Class type) { } public ObjectStreamSerializer(TypeResolver typeResolver, Class type) { - super(typeResolver, type, createObjectStreamCreator(type)); + super(typeResolver, type, createObjectStreamCreator(typeResolver, type)); if (!Serializable.class.isAssignableFrom(type)) { throw new IllegalArgumentException( String.format("Class %s should implement %s.", type, Serializable.class)); } - Serializer fallbackSerializer = fallbackSerializer(typeResolver, type); - this.fallbackSerializer = fallbackSerializer; - if (fallbackSerializer != null) { - slotsInfos = new SlotInfo[0]; - return; - } if (!Throwable.class.isAssignableFrom(type)) { LOG.warn( "{} customized jdk serialization, which is inefficient. " @@ -223,79 +215,14 @@ public ObjectStreamSerializer(TypeResolver typeResolver, Class type) { slotsInfos = slotsInfoList.toArray(new SlotInfo[0]); } - private static Serializer fallbackSerializer(TypeResolver typeResolver, Class type) { - if (JdkVersion.MAJOR_VERSION < 25) { - return null; - } - Class childSerializerClass = null; - if (Collection.class.isAssignableFrom(type)) { - childSerializerClass = ChildContainerSerializers.getCollectionSerializerClass(type); - } else if (Map.class.isAssignableFrom(type)) { - childSerializerClass = ChildContainerSerializers.getMapSerializerClass(type); - } - if (childSerializerClass != null) { - return Serializers.newSerializer(typeResolver, type, childSerializerClass); - } - if (type.getName().startsWith("java.")) { - return new JavaSerializer(typeResolver, type); - } - return null; - } - /** Creates an ObjectCreator for Java ObjectStream-compatible reconstruction. */ - private static ObjectCreator createObjectStreamCreator(Class type) { - if (AndroidSupport.IS_ANDROID) { - return ObjectCreators.getObjectCreator(type); - } - if (JdkVersion.MAJOR_VERSION >= 25) { - if (hasJdk25Fallback(type)) { - return new FallbackOnlyObjectCreator<>(type); - } - // ObjectStreamSerializer must preserve Java serialization construction semantics. On JDK25+ - // this path cannot fall back to Unsafe, including inside GraalVM native images. - return new ObjectCreators.ParentNoArgCtrObjectCreator<>(type); - } - return ObjectCreators.getObjectCreator(type); - } - - private static boolean hasJdk25Fallback(Class type) { - if (JdkVersion.MAJOR_VERSION < 25) { - return false; - } - if (type.getName().startsWith("java.")) { - return true; - } - if (Collection.class.isAssignableFrom(type)) { - return ChildContainerSerializers.getCollectionSerializerClass(type) != null; - } - if (Map.class.isAssignableFrom(type)) { - return ChildContainerSerializers.getMapSerializerClass(type) != null; - } - return false; - } - - private static final class FallbackOnlyObjectCreator extends ObjectCreator { - private FallbackOnlyObjectCreator(Class type) { - super(type); - } - - @Override - public T newInstance() { - throw new ForyException("ObjectStreamSerializer fallback owns construction for " + type); - } - - @Override - public T newInstanceWithArguments(Object... arguments) { - throw new ForyException("ObjectStreamSerializer fallback owns construction for " + type); - } + private static ObjectCreator createObjectStreamCreator( + TypeResolver typeResolver, Class type) { + return typeResolver.getObjectCreator(type); } @Override public void write(WriteContext writeContext, Object value) { - if (fallbackSerializer != null) { - fallbackSerializer.write(writeContext, value); - return; - } MemoryBuffer buffer = writeContext.getBuffer(); buffer.writeInt16((short) slotsInfos.length); try { @@ -348,9 +275,6 @@ public void write(WriteContext writeContext, Object value) { @Override public Object read(ReadContext readContext) { - if (fallbackSerializer != null) { - return fallbackSerializer.read(readContext); - } MemoryBuffer buffer = readContext.getBuffer(); Object obj = objectCreator.newInstance(); readContext.reference(obj); @@ -410,43 +334,14 @@ public Object read(ReadContext readContext) { Method readObjectMethod = streamTypeInfo.readObjectMethod; if (readObjectMethod == null) { - // For standard field serialization - use getCurrentReadSerializer() - matchedSlot.getCurrentReadSerializer().readAndSetFields(readContext, obj); - } else { - // For custom readObject, it handles its own format - ForyStructInputStream objectInputStream = matchedSlot.getObjectInputStream(); - MemoryBuffer oldBuffer = objectInputStream.buffer; - Object oldObject = objectInputStream.targetObject; - ReadContext oldReadContext = objectInputStream.readContext; - ForyStructInputStream.GetFieldImpl oldGetField = objectInputStream.getField; - ForyStructInputStream.GetFieldImpl getField = - (ForyStructInputStream.GetFieldImpl) matchedSlot.getFieldPool().popOrNull(); - if (getField == null) { - getField = new ForyStructInputStream.GetFieldImpl(matchedSlot); - } - boolean fieldsRead = objectInputStream.fieldsRead; - try { - objectInputStream.fieldsRead = false; - objectInputStream.buffer = buffer; - objectInputStream.targetObject = obj; - objectInputStream.readContext = readContext; - objectInputStream.getField = getField; - objectInputStream.callbacks = callbacks; - if (streamTypeInfo.readObjectFunc != null) { - streamTypeInfo.readObjectFunc.accept(obj, objectInputStream); - } else { - readObjectMethod.invoke(obj, objectInputStream); - } - } finally { - objectInputStream.fieldsRead = fieldsRead; - objectInputStream.buffer = oldBuffer; - objectInputStream.targetObject = oldObject; - objectInputStream.readContext = oldReadContext; - objectInputStream.getField = oldGetField; - matchedSlot.getFieldPool().add(getField); - objectInputStream.callbacks = null; - Arrays.fill(getField.vals, ForyStructInputStream.NO_VALUE_STUB); + if (streamTypeInfo.defaultReadObjectHandle != null) { + readObjectStreamSlot(matchedSlot, readContext, obj, callbacks, true); + } else { + // For standard field serialization - use getCurrentReadSerializer() + matchedSlot.getCurrentReadSerializer().readAndSetFields(readContext, obj); } + } else { + readObjectStreamSlot(matchedSlot, readContext, obj, callbacks, false); } } @@ -467,18 +362,129 @@ public Object read(ReadContext readContext) { for (ObjectInputValidation validation : callbacks.values()) { validation.validateObject(); } - } catch (InvocationTargetException | IllegalAccessException | InvalidObjectException e) { + } catch (InvocationTargetException + | IllegalAccessException + | IOException + | ClassNotFoundException e) { throwSerializationException(type, e); } return obj; } + private void readObjectStreamSlot( + SlotInfo matchedSlot, + ReadContext readContext, + Object obj, + TreeMap callbacks, + boolean defaultRead) + throws IOException, + ClassNotFoundException, + InvocationTargetException, + IllegalAccessException { + StreamTypeInfo streamTypeInfo = matchedSlot.getStreamTypeInfo(); + ForyStructInputStream objectInputStream = matchedSlot.getObjectInputStream(); + MemoryBuffer oldBuffer = objectInputStream.buffer; + Object oldObject = objectInputStream.targetObject; + ReadContext oldReadContext = objectInputStream.readContext; + Object[] oldFieldValuesOverride = objectInputStream.fieldValuesOverride; + ForyStructInputStream.GetFieldImpl oldGetField = objectInputStream.getField; + ForyStructInputStream.GetFieldImpl getField = + (ForyStructInputStream.GetFieldImpl) matchedSlot.getFieldPool().popOrNull(); + if (getField == null) { + getField = new ForyStructInputStream.GetFieldImpl(matchedSlot); + } + boolean fieldsRead = objectInputStream.fieldsRead; + try { + objectInputStream.fieldsRead = false; + objectInputStream.buffer = readContext.getBuffer(); + objectInputStream.targetObject = obj; + objectInputStream.readContext = readContext; + objectInputStream.fieldValuesOverride = null; + objectInputStream.getField = getField; + objectInputStream.callbacks = callbacks; + if (defaultRead) { + objectInputStream.runDefaultReadObject(); + } else if (streamTypeInfo.readObjectFunc != null) { + streamTypeInfo.readObjectFunc.accept(obj, objectInputStream); + } else { + streamTypeInfo.readObjectMethod.invoke(obj, objectInputStream); + } + } finally { + objectInputStream.fieldsRead = fieldsRead; + objectInputStream.buffer = oldBuffer; + objectInputStream.targetObject = oldObject; + objectInputStream.readContext = oldReadContext; + objectInputStream.fieldValuesOverride = oldFieldValuesOverride; + objectInputStream.getField = oldGetField; + matchedSlot.getFieldPool().add(getField); + objectInputStream.callbacks = null; + Arrays.fill(getField.vals, ForyStructInputStream.NO_VALUE_STUB); + } + } + @Override public Object copy(CopyContext copyContext, Object value) { - if (fallbackSerializer != null) { - return fallbackSerializer.copy(copyContext, value); + if (!canCopyWithDefaultReadObject()) { + return super.copy(copyContext, value); + } + Object copy = objectCreator.newInstance(); + copyContext.reference(value, copy); + try { + for (SlotInfo slotInfo : slotsInfos) { + copyObjectStreamSlot(copyContext, value, copy, slotInfo); + } + return copy; + } catch (IOException | ClassNotFoundException e) { + throw new ForyException("Failed to copy Java serialization fields for " + type, e); + } + } + + private boolean canCopyWithDefaultReadObject() { + for (SlotInfo slotInfo : slotsInfos) { + if (slotInfo.getStreamTypeInfo().defaultReadObjectHandle == null) { + return false; + } + } + return true; + } + + private void copyObjectStreamSlot( + CopyContext copyContext, Object sourceObject, Object targetObject, SlotInfo slotInfo) + throws IOException, ClassNotFoundException { + ForyStructInputStream objectInputStream = slotInfo.getObjectInputStream(); + Object oldObject = objectInputStream.targetObject; + ReadContext oldReadContext = objectInputStream.readContext; + Object[] oldFieldValuesOverride = objectInputStream.fieldValuesOverride; + ForyStructInputStream.GetFieldImpl oldGetField = objectInputStream.getField; + ForyStructInputStream.GetFieldImpl getField = + (ForyStructInputStream.GetFieldImpl) slotInfo.getFieldPool().popOrNull(); + if (getField == null) { + getField = new ForyStructInputStream.GetFieldImpl(slotInfo); + } + boolean fieldsRead = objectInputStream.fieldsRead; + try { + objectInputStream.fieldsRead = false; + objectInputStream.targetObject = targetObject; + objectInputStream.readContext = null; + objectInputStream.fieldValuesOverride = + slotInfo + .getSlotsSerializer() + .copyFieldValuesForPutFields( + copyContext, + sourceObject, + slotInfo.getFieldIndexMap(), + slotInfo.getNumPutFields()); + objectInputStream.getField = getField; + objectInputStream.runDefaultReadObject(); + } finally { + objectInputStream.fieldsRead = fieldsRead; + objectInputStream.targetObject = oldObject; + objectInputStream.readContext = oldReadContext; + objectInputStream.fieldValuesOverride = oldFieldValuesOverride; + objectInputStream.getField = oldGetField; + slotInfo.getFieldPool().add(getField); + Arrays.fill(getField.vals, ForyStructInputStream.NO_VALUE_STUB); } - return super.copy(copyContext, value); } /** @@ -680,6 +686,7 @@ private static class StreamTypeInfo { private final Method writeObjectMethod; private final Method readObjectMethod; private final Method readObjectNoData; + private final MethodHandle defaultReadObjectHandle; private final BiConsumer writeObjectFunc; private final BiConsumer readObjectFunc; private final Consumer readObjectNoDataFunc; @@ -711,6 +718,10 @@ private StreamTypeInfo(Class type) { this.writeObjectMethod = writeMethod; this.readObjectMethod = readMethod; this.readObjectNoData = noDataMethod; + this.defaultReadObjectHandle = + AndroidSupport.IS_ANDROID || GraalvmSupport.IN_GRAALVM_NATIVE_IMAGE + ? null + : _JDKAccess.getSerializationDefaultReadObjectHandle(type); if (AndroidSupport.IS_ANDROID) { makeAccessible(writeObjectMethod); makeAccessible(readObjectMethod); @@ -842,7 +853,8 @@ public SlotsInfo(TypeResolver typeResolver, Class type) { fieldIndexMap = new ObjectIntMap<>(4, 0.4f); if (streamTypeInfo != null && (streamTypeInfo.writeObjectMethod != null - || streamTypeInfo.readObjectMethod != null)) { + || streamTypeInfo.readObjectMethod != null + || streamTypeInfo.defaultReadObjectHandle != null)) { this.numPutFields = slotsSerializer.getNumFields(); this.putFieldTypes = new Class[numPutFields]; slotsSerializer.populateFieldInfo(fieldIndexMap, putFieldTypes); @@ -861,7 +873,9 @@ public SlotsInfo(TypeResolver typeResolver, Class type) { } else { objectOutputStream = null; } - if (streamTypeInfo != null && streamTypeInfo.readObjectMethod != null) { + if (streamTypeInfo != null + && (streamTypeInfo.readObjectMethod != null + || streamTypeInfo.defaultReadObjectHandle != null)) { try { objectInputStream = new ForyStructInputStream(this); } catch (IOException e) { @@ -1317,6 +1331,7 @@ private static class ForyStructInputStream extends ObjectInputStream { private MemoryBuffer buffer; private Object targetObject; private GetFieldImpl getField; + private Object[] fieldValuesOverride; private boolean fieldsRead; private TreeMap callbacks; @@ -1469,8 +1484,11 @@ public GetField readFields() throws IOException { if (fieldsRead) { throw new NotActiveException("not in readObject invocation or fields already read"); } - // Read field values using MetaShare serialization - Object[] vals = slotsInfo.getCurrentReadSerializer().readFieldValues(readContext); + Object[] vals = fieldValuesOverride; + if (vals == null) { + // Read field values using MetaShare serialization + vals = slotsInfo.getCurrentReadSerializer().readFieldValues(readContext); + } System.arraycopy(vals, 0, getField.vals, 0, vals.length); fieldsRead = true; return getField; @@ -1509,11 +1527,29 @@ public void defaultReadObject() throws IOException, ClassNotFoundException { if (fieldsRead) { throw new NotActiveException("not in readObject invocation or fields already read"); } - // Read fields using MetaShare serialization (layer meta already read by getReadSerializer()) - slotsInfo.getCurrentReadSerializer().readAndSetFields(readContext, targetObject); + runDefaultReadObject(); fieldsRead = true; } + private void runDefaultReadObject() throws IOException, ClassNotFoundException { + MethodHandle defaultReadObjectHandle = slotsInfo.getStreamTypeInfo().defaultReadObjectHandle; + if (defaultReadObjectHandle == null) { + // Read fields using MetaShare serialization (layer meta already read by + // getReadSerializer()) + slotsInfo.getCurrentReadSerializer().readAndSetFields(readContext, targetObject); + return; + } + try { + // This public Java serialization compatibility hook writes final fields without requiring + // per-package JDK opens. It calls this stream's readFields() implementation for values. + defaultReadObjectHandle.invoke(targetObject, this); + } catch (IOException | ClassNotFoundException e) { + throw e; + } catch (Throwable e) { + throw new IOException("Failed to default-read Java serialization fields", e); + } + } + // At `registerValidation` point int `readObject` root is only partially correct. To fully // restore it user may need access to other state which is created by the subclass and // at this point will be null. Users thus use `registerValidation`. diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/ReplaceResolveSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/ReplaceResolveSerializer.java index 525287521b..22431bdbbf 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/ReplaceResolveSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/ReplaceResolveSerializer.java @@ -37,9 +37,7 @@ import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.platform.AndroidSupport; import org.apache.fory.platform.GraalvmSupport; -import org.apache.fory.platform.JdkVersion; import org.apache.fory.platform.internal._JDKAccess; -import org.apache.fory.reflect.ObjectCreators; import org.apache.fory.resolver.ClassResolver; import org.apache.fory.resolver.TypeInfo; import org.apache.fory.resolver.TypeResolver; @@ -251,18 +249,11 @@ private static Class dataSerializerClass( serializerClass = ExternalizableSerializer.class; } else if (JavaSerializer.getReadRefMethod(cls, true) == null && JavaSerializer.getWriteObjectMethod(cls, true) == null) { - if (JdkVersion.MAJOR_VERSION >= 25 - && Serializable.class.isAssignableFrom(cls) - && !ObjectCreators.supportsJdk25Creation(cls)) { - serializerClass = typeResolver.getDefaultJDKStreamSerializerType(); - } else { - serializerClass = - classResolver.getObjectSerializerClass( - cls, - sc -> - methodInfoCache.setObjectSerializer( - createDataSerializer(typeResolver, cls, sc))); - } + serializerClass = + classResolver.getObjectSerializerClass( + cls, + sc -> + methodInfoCache.setObjectSerializer(createDataSerializer(typeResolver, cls, sc))); } else { serializerClass = typeResolver.getDefaultJDKStreamSerializerType(); } diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/Serializers.java b/java/fory-core/src/main/java/org/apache/fory/serializer/Serializers.java index 209f76db34..85a091516d 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/Serializers.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/Serializers.java @@ -654,7 +654,7 @@ private static void checkStringTokenizerAccess() { private static UnsupportedOperationException stringTokenizerAccessError() { return new UnsupportedOperationException( "StringTokenizer serialization requires JDK internal field access. On JDK25+, open " - + "java.base/java.util to org.apache.fory.core,org.apache.fory.format."); + + "java.base/java.lang.invoke to org.apache.fory.core."); } private static final class Accessors { diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/StaticGeneratedStructSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/StaticGeneratedStructSerializer.java index e3990ac7c7..53b9a23df1 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/StaticGeneratedStructSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/StaticGeneratedStructSerializer.java @@ -20,6 +20,8 @@ package org.apache.fory.serializer; import java.lang.reflect.Constructor; +import java.lang.reflect.Field; +import java.lang.reflect.Modifier; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -43,6 +45,7 @@ import org.apache.fory.serializer.FieldGroups.SerializationFieldInfo; import org.apache.fory.serializer.converter.FieldConverters; import org.apache.fory.type.Descriptor; +import org.apache.fory.type.DescriptorBuilder; import org.apache.fory.type.DescriptorGrouper; import org.apache.fory.util.StringUtils; @@ -172,7 +175,25 @@ public final FieldGroups buildLocalFieldGroups(List descriptors) { } protected final List runtimeDescriptors(List descriptors) { - return typeResolver.normalizeFieldDescriptors(type, true, descriptors); + return typeResolver.normalizeFieldDescriptors(type, true, attachFields(descriptors)); + } + + private List attachFields(List descriptors) { + Map fields = new HashMap<>(); + for (Field field : Descriptor.getFields(type)) { + fields.put(field.getDeclaringClass().getName() + "." + field.getName(), field); + } + List result = new ArrayList<>(descriptors.size()); + for (Descriptor descriptor : descriptors) { + if (descriptor.getField() != null || !Modifier.isFinal(descriptor.getModifier())) { + result.add(descriptor); + continue; + } + Field field = fields.get(fieldKey(descriptor)); + result.add( + field == null ? descriptor : new DescriptorBuilder(descriptor).field(field).build()); + } + return result; } private static boolean hasSourceOnlyMetadata(List descriptors) { @@ -227,6 +248,104 @@ public final int[] localFieldIds( return ids; } + protected final int[] buildConstructorFieldIds(List descriptors) { + String[] fieldNames = objectCreator.getConstructorFieldNames(); + if (fieldNames.length == 0) { + return null; + } + Class[] declaringClasses = objectCreator.getConstructorFieldDeclaringClasses(); + boolean[] finalFields = objectCreator.getConstructorFieldFinal(); + int[] ids = new int[fieldNames.length]; + for (int i = 0; i < fieldNames.length; i++) { + Class declaringClass = declaringClasses == null ? null : declaringClasses[i]; + ids[i] = constructorFieldId(descriptors, declaringClass, fieldNames[i], !finalFields[i]); + } + return ids; + } + + private int constructorFieldId( + List descriptors, + Class declaringClass, + String fieldName, + boolean allowMissing) { + int id = UNKNOWN_FIELD; + String declaringClassName = declaringClass == null ? null : declaringClass.getName(); + for (int i = 0; i < descriptors.size(); i++) { + Descriptor descriptor = descriptors.get(i); + if (!descriptor.getName().equals(fieldName) + || (declaringClassName != null + && !descriptor.getDeclaringClass().equals(declaringClassName))) { + continue; + } + if (id != UNKNOWN_FIELD) { + throw new ForyException( + "Constructor field " + fieldName + " is ambiguous because multiple fields match"); + } + id = i; + } + if (id == UNKNOWN_FIELD && !allowMissing) { + throw new ForyException("Constructor field " + fieldName + " is not serialized"); + } + return id; + } + + protected final long[] buildConstructorFieldBits(int size, int[] indexes) { + if (indexes == null) { + return null; + } + long[] bits = newFieldBits(size); + for (int index : indexes) { + if (index >= 0) { + markField(bits, index); + } + } + return bits; + } + + protected static long[] newFieldBits(int size) { + return new long[(size + Long.SIZE - 1) / Long.SIZE]; + } + + protected static boolean hasField(long[] bits, int fieldId) { + return (bits[fieldId / Long.SIZE] & (1L << (fieldId % Long.SIZE))) != 0; + } + + protected static void markField(long[] bits, int fieldId) { + bits[fieldId / Long.SIZE] |= 1L << (fieldId % Long.SIZE); + } + + protected static int countConstructorFields(long[] constructorFieldBits) { + int count = 0; + for (long constructorFields : constructorFieldBits) { + count += Long.bitCount(constructorFields); + } + return count; + } + + protected final void setGeneratedFieldValue( + Object targetObject, SerializationFieldInfo fieldInfo, Object fieldValue) { + if (fieldInfo.fieldAccessor != null) { + fieldInfo.fieldAccessor.putObject(targetObject, fieldValue); + return; + } + if (fieldInfo.fieldConverter != null) { + fieldInfo.fieldConverter.set(targetObject, fieldValue); + return; + } + throw new ForyException("Generated field " + fieldInfo.getName() + " is not writable"); + } + + protected final Object copyConstructorFieldValue( + CopyContext copyContext, + Object originObject, + Object fieldValue, + SerializationFieldInfo fieldInfo) { + if (fieldValue == originObject) { + throwConstructorCycle(type); + } + return copyFieldValue(copyContext, fieldValue, fieldInfo); + } + protected final void writeBuildInFieldValue( WriteContext writeContext, SerializationFieldInfo fieldInfo, Object fieldValue) { // Some schema-built-in fields still use container-shaped Java accessors, such as diff --git a/java/fory-core/src/main/java25/module-info.java b/java/fory-core/src/main/java25/module-info.java new file mode 100644 index 0000000000..25f4419f1f --- /dev/null +++ b/java/fory-core/src/main/java25/module-info.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +module org.apache.fory.core { + requires java.logging; + + requires static java.sql; + requires static com.google.common; + requires static org.slf4j; + requires static jsr305; + requires static jdk.incubator.vector; + + exports org.apache.fory; + exports org.apache.fory.annotation; + exports org.apache.fory.builder; + exports org.apache.fory.codegen; + exports org.apache.fory.collection; + exports org.apache.fory.config; + exports org.apache.fory.context; + exports org.apache.fory.exception; + exports org.apache.fory.io; + exports org.apache.fory.logging; + exports org.apache.fory.memory; + exports org.apache.fory.meta; + exports org.apache.fory.platform; + exports org.apache.fory.pool; + exports org.apache.fory.reflect; + exports org.apache.fory.resolver; + exports org.apache.fory.serializer; + exports org.apache.fory.serializer.collection; + exports org.apache.fory.serializer.converter; + exports org.apache.fory.serializer.scala; + exports org.apache.fory.serializer.struct; + exports org.apache.fory.type; + exports org.apache.fory.type.union; + exports org.apache.fory.type.unsigned; + exports org.apache.fory.util; + exports org.apache.fory.util.function; + exports org.apache.fory.util.record; + + // Sibling artifacts still use these internal JDK access helpers; keep this qualified + // export until those imports move behind public owner APIs. + exports org.apache.fory.platform.internal to + org.apache.fory.extension, + org.apache.fory.format; +} diff --git a/java/fory-core/src/main/java25/org/apache/fory/builder/UnsafeCodegenSupport.java b/java/fory-core/src/main/java25/org/apache/fory/builder/UnsafeCodegenSupport.java new file mode 100644 index 0000000000..44aad39bb0 --- /dev/null +++ b/java/fory-core/src/main/java25/org/apache/fory/builder/UnsafeCodegenSupport.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.builder; + +import java.lang.reflect.Field; +import org.apache.fory.annotation.Internal; + +/** JDK25 replacement that keeps builder classes free of Unsafe runtime linkage. */ +@Internal +public final class UnsafeCodegenSupport { + private UnsafeCodegenSupport() {} + + public static Object unsafe() { + throw unsupported(); + } + + public static long objectFieldOffset(Field field) { + throw unsupported(); + } + + static String unsafeTypeName() { + throw unsupported(); + } + + static String unsafeInitCode() { + throw unsupported(); + } + + private static UnsupportedOperationException unsupported() { + return new UnsupportedOperationException("Generated Unsafe access is unsupported on JDK25+"); + } +} diff --git a/java/fory-core/src/main/java25/org/apache/fory/platform/internal/DefineClass.java b/java/fory-core/src/main/java25/org/apache/fory/platform/internal/DefineClass.java index 2c4e62358f..4dea13dfcf 100644 --- a/java/fory-core/src/main/java25/org/apache/fory/platform/internal/DefineClass.java +++ b/java/fory-core/src/main/java25/org/apache/fory/platform/internal/DefineClass.java @@ -93,17 +93,11 @@ public static Class defineHiddenNestmate(Class neighbor, byte[] bytecodes) .defineHiddenClass(bytecodes, true, Lookup.ClassOption.NESTMATE) .lookupClass(); } catch (IllegalAccessException | IllegalStateException e) { - Module module = neighbor.getModule(); - Package pkg = neighbor.getPackage(); - String packageName = pkg == null ? "" : pkg.getName(); throw new IllegalStateException( "Cannot define hidden nestmate for " + neighbor.getName() - + " because package " - + packageName - + " in module " - + module.getName() - + " is not open to org.apache.fory.core,org.apache.fory.format", + + ". JDK25 zero-Unsafe mode requires java.base/java.lang.invoke to be open to " + + "org.apache.fory.core", e); } } diff --git a/java/fory-core/src/main/java25/org/apache/fory/platform/internal/_JDKAccess.java b/java/fory-core/src/main/java25/org/apache/fory/platform/internal/_JDKAccess.java index 45ac0a3790..e4153744cc 100644 --- a/java/fory-core/src/main/java25/org/apache/fory/platform/internal/_JDKAccess.java +++ b/java/fory-core/src/main/java25/org/apache/fory/platform/internal/_JDKAccess.java @@ -19,6 +19,8 @@ package org.apache.fory.platform.internal; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; import java.lang.invoke.CallSite; import java.lang.invoke.LambdaConversionException; import java.lang.invoke.LambdaMetafactory; @@ -30,6 +32,7 @@ import java.lang.invoke.VarHandle; import java.lang.reflect.Field; import java.lang.reflect.Method; +import java.lang.reflect.Modifier; import java.lang.reflect.Proxy; import java.nio.charset.StandardCharsets; import java.util.HashMap; @@ -55,26 +58,18 @@ import org.apache.fory.util.function.ToCharFunction; import org.apache.fory.util.function.ToFloatFunction; import org.apache.fory.util.function.ToShortFunction; -import sun.misc.Unsafe; /** JDK internals access for the JDK25 multi-release runtime. */ // CHECKSTYLE.OFF:TypeName public class _JDKAccess { // CHECKSTYLE.ON:TypeName public static final boolean IS_OPEN_J9; - public static final Unsafe UNSAFE = null; public static final boolean JDK_INTERNAL_FIELD_ACCESS; public static final boolean JDK_LANG_FIELD_ACCESS; public static final boolean JDK_STRING_FIELD_ACCESS; public static final boolean JDK_COLLECTION_FIELD_ACCESS; public static final boolean JDK_CONCURRENT_FIELD_ACCESS; public static final boolean JDK_PROXY_FIELD_ACCESS; - public static final Class _INNER_UNSAFE_CLASS = null; - public static final Object _INNER_UNSAFE = null; - - public static Unsafe unsafe() { - return UNSAFE; - } private static final ClassValueCache lookupCache = ClassValueCache.newClassKeyCache(32); @@ -115,12 +110,12 @@ public static Unsafe unsafe() { StringHandles stringHandles = initStringHandles(valueField.getType(), countField, offsetField); - JDK_LANG_FIELD_ACCESS = canOpen(String.class); + JDK_LANG_FIELD_ACCESS = canAccess(String.class); JDK_STRING_FIELD_ACCESS = stringHandles != null; - JDK_COLLECTION_FIELD_ACCESS = canOpen("java.util.Collections$SynchronizedCollection"); + JDK_COLLECTION_FIELD_ACCESS = canAccess("java.util.Collections$SynchronizedCollection"); JDK_CONCURRENT_FIELD_ACCESS = - canOpen(ArrayBlockingQueue.class) && canOpen(LinkedBlockingQueue.class); - JDK_PROXY_FIELD_ACCESS = canOpen(Proxy.class); + canAccess(ArrayBlockingQueue.class) && canAccess(LinkedBlockingQueue.class); + JDK_PROXY_FIELD_ACCESS = canAccess(Proxy.class); JDK_INTERNAL_FIELD_ACCESS = JDK_STRING_FIELD_ACCESS; STRING_VALUE_HANDLE = stringHandles == null ? null : stringHandles.value; @@ -143,7 +138,7 @@ private static Field getStringFieldNullable(String fieldName) { private static StringHandles initStringHandles( Class stringValueType, Field countField, Field offsetField) { try { - Lookup stringLookup = MethodHandles.privateLookupIn(String.class, MethodHandles.lookup()); + Lookup stringLookup = _Lookup._trustedLookup(String.class); return new StringHandles( stringLookup.findVarHandle(String.class, "value", stringValueType), STRING_VALUE_FIELD_IS_BYTES @@ -158,17 +153,17 @@ private static StringHandles initStringHandles( } } - private static boolean canOpen(String className) { + private static boolean canAccess(String className) { try { - return canOpen(Class.forName(className)); + return canAccess(Class.forName(className)); } catch (Throwable ignored) { return false; } } - private static boolean canOpen(Class type) { + private static boolean canAccess(Class type) { try { - MethodHandles.privateLookupIn(type, MethodHandles.lookup()); + _Lookup._trustedLookup(type); return true; } catch (Throwable ignored) { return false; @@ -248,7 +243,7 @@ public static MethodHandle readResolveHandle(Class cls, Method method) } catch (RuntimeException inaccessible) { throw new IllegalStateException( "SerializedLambda readResolve requires java.base/java.lang.invoke to be open to " - + "org.apache.fory.core,org.apache.fory.format", + + "org.apache.fory.core", inaccessible); } return MethodHandles.lookup().unreflect(method); @@ -431,89 +426,97 @@ private static void checkStringAccess(String target) { if (!JDK_STRING_FIELD_ACCESS) { throw new UnsupportedOperationException( target - + " private access is unavailable; open java.base/java.lang to " - + "org.apache.fory.core,org.apache.fory.format"); - } - } - - private static class SerializationMethods { - private static final Object REFLECTION_FACTORY; - private static final Method WRITE_OBJECT; - private static final Method READ_OBJECT; - private static final Method READ_OBJECT_NO_DATA; - private static final Method WRITE_REPLACE; - private static final Method READ_RESOLVE; - - static { - Object reflectionFactory = null; - Method writeObject = null; - Method readObject = null; - Method readObjectNoData = null; - Method writeReplace = null; - Method readResolve = null; - try { - Class factoryClass = Class.forName("sun.reflect.ReflectionFactory"); - Method getReflectionFactory = factoryClass.getDeclaredMethod("getReflectionFactory"); - reflectionFactory = getReflectionFactory.invoke(null); - writeObject = factoryClass.getDeclaredMethod("writeObjectForSerialization", Class.class); - readObject = factoryClass.getDeclaredMethod("readObjectForSerialization", Class.class); - readObjectNoData = - factoryClass.getDeclaredMethod("readObjectNoDataForSerialization", Class.class); - writeReplace = factoryClass.getDeclaredMethod("writeReplaceForSerialization", Class.class); - readResolve = factoryClass.getDeclaredMethod("readResolveForSerialization", Class.class); - } catch (Throwable e) { - ExceptionUtils.ignore(e); - } - REFLECTION_FACTORY = reflectionFactory; - WRITE_OBJECT = writeObject; - READ_OBJECT = readObject; - READ_OBJECT_NO_DATA = readObjectNoData; - WRITE_REPLACE = writeReplace; - READ_RESOLVE = readResolve; + + " private access is unavailable; open java.base/java.lang.invoke to " + + "org.apache.fory.core"); } } - private static Method getSerializationMethod(Class type, Method factoryMethod) { - if (!isSerializationHookLookupAvailable() || factoryMethod == null) { - return null; + private static final ClassValueCache serializationMethodsCache = + ClassValueCache.newClassKeyCache(32); + + private static final class SerializationMethods { + private final Method writeObject; + private final Method readObject; + private final Method readObjectNoData; + private final Method writeReplace; + private final Method readResolve; + + private SerializationMethods(Class type) { + writeObject = + getPrivateMethod(type, "writeObject", void.class, ObjectOutputStream.class); + readObject = getPrivateMethod(type, "readObject", void.class, ObjectInputStream.class); + readObjectNoData = getPrivateMethod(type, "readObjectNoData", void.class); + writeReplace = getInheritableObjectMethod(type, "writeReplace"); + readResolve = getInheritableObjectMethod(type, "readResolve"); } + } + + private static SerializationMethods serializationMethods(Class type) { + return serializationMethodsCache.get(type, () -> new SerializationMethods(type)); + } + + private static Method getPrivateMethod( + Class type, String methodName, Class returnType, Class... parameterTypes) { try { - MethodHandle handle = - (MethodHandle) factoryMethod.invoke(SerializationMethods.REFLECTION_FACTORY, type); - return handle == null ? null : MethodHandles.reflectAs(Method.class, handle); - } catch (Throwable e) { + Method method = type.getDeclaredMethod(methodName, parameterTypes); + int modifiers = method.getModifiers(); + if (Modifier.isPrivate(modifiers) + && !Modifier.isStatic(modifiers) + && method.getReturnType() == returnType) { + return method; + } + } catch (NoSuchMethodException | SecurityException e) { ExceptionUtils.ignore(e); - return null; } + return null; + } + + private static Method getInheritableObjectMethod(Class type, String methodName) { + Class cls = type; + while (cls != null) { + try { + Method method = cls.getDeclaredMethod(methodName); + int modifiers = method.getModifiers(); + if (!Modifier.isStatic(modifiers) && method.getReturnType() == Object.class) { + return method; + } + return null; + } catch (NoSuchMethodException e) { + ExceptionUtils.ignore(e); + } catch (SecurityException e) { + return null; + } + cls = cls.getSuperclass(); + } + return null; } public static Method getSerializationWriteObjectMethod(Class type) { - return getSerializationMethod(type, SerializationMethods.WRITE_OBJECT); + return serializationMethods(type).writeObject; } public static Method getSerializationReadObjectMethod(Class type) { - return getSerializationMethod(type, SerializationMethods.READ_OBJECT); + return serializationMethods(type).readObject; } public static Method getSerializationReadObjectNoDataMethod(Class type) { - return getSerializationMethod(type, SerializationMethods.READ_OBJECT_NO_DATA); + return serializationMethods(type).readObjectNoData; + } + + public static MethodHandle getSerializationDefaultReadObjectHandle(Class type) { + return null; } public static Method getSerializationWriteReplaceMethod(Class type) { - return getSerializationMethod(type, SerializationMethods.WRITE_REPLACE); + return serializationMethods(type).writeReplace; } public static Method getSerializationReadResolveMethod(Class type) { - return getSerializationMethod(type, SerializationMethods.READ_RESOLVE); + return serializationMethods(type).readResolve; } public static boolean isSerializationHookLookupAvailable() { - return SerializationMethods.REFLECTION_FACTORY != null - && SerializationMethods.WRITE_OBJECT != null - && SerializationMethods.READ_OBJECT != null - && SerializationMethods.READ_OBJECT_NO_DATA != null - && SerializationMethods.WRITE_REPLACE != null - && SerializationMethods.READ_RESOLVE != null; + return true; } public static T tryMakeFunction( diff --git a/java/fory-core/src/main/java25/org/apache/fory/platform/internal/_Lookup.java b/java/fory-core/src/main/java25/org/apache/fory/platform/internal/_Lookup.java index b7a0b8272f..66a075b729 100644 --- a/java/fory-core/src/main/java25/org/apache/fory/platform/internal/_Lookup.java +++ b/java/fory-core/src/main/java25/org/apache/fory/platform/internal/_Lookup.java @@ -19,31 +19,22 @@ package org.apache.fory.platform.internal; -import java.lang.invoke.MethodHandles; import java.lang.invoke.MethodHandles.Lookup; +import java.lang.reflect.Field; // CHECKSTYLE.OFF:TypeName class _Lookup { // CHECKSTYLE.ON:TypeName - static final Lookup IMPL_LOOKUP = MethodHandles.lookup(); + private static volatile Lookup implLookup; // CHECKSTYLE.OFF:MethodName public static Lookup _trustedLookup(Class objectClass) { // CHECKSTYLE.ON:MethodName - return privateLookupIn(objectClass, MethodHandles.lookup()); + return implLookup().in(objectClass); } public static Lookup privateLookupIn(Class targetClass, Lookup caller) { - try { - Module foryModule = _Lookup.class.getModule(); - Module targetModule = targetClass.getModule(); - if (foryModule != targetModule) { - foryModule.addReads(targetModule); - } - return MethodHandles.privateLookupIn(targetClass, caller); - } catch (IllegalAccessException e) { - throw new IllegalStateException(privateAccessMessage(targetClass), e); - } + return _trustedLookup(targetClass); } /** @@ -55,20 +46,36 @@ public static Class defineClass(Lookup lookup, byte[] bytes) { try { return lookup.defineClass(bytes); } catch (IllegalAccessException e) { - throw new IllegalStateException(privateAccessMessage(lookup.lookupClass()), e); + throw new IllegalStateException(trustedLookupMessage(), e); + } + } + + private static Lookup loadImplLookup() { + try { + Field field = Lookup.class.getDeclaredField("IMPL_LOOKUP"); + field.setAccessible(true); + return (Lookup) field.get(null); + } catch (ReflectiveOperationException | RuntimeException e) { + throw new IllegalStateException(trustedLookupMessage(), e); + } + } + + private static Lookup implLookup() { + Lookup lookup = implLookup; + if (lookup == null) { + synchronized (_Lookup.class) { + lookup = implLookup; + if (lookup == null) { + lookup = loadImplLookup(); + implLookup = lookup; + } + } } + return lookup; } - private static String privateAccessMessage(Class targetClass) { - Module module = targetClass.getModule(); - Package pkg = targetClass.getPackage(); - String packageName = pkg == null ? "" : pkg.getName(); - return "Private lookup for " - + targetClass.getName() - + " requires package " - + packageName - + " in module " - + module.getName() - + " to be open to org.apache.fory.core,org.apache.fory.format"; + private static String trustedLookupMessage() { + return "JDK25 zero-Unsafe mode requires java.base/java.lang.invoke to be open to " + + "org.apache.fory.core"; } } diff --git a/java/fory-core/src/main/java25/org/apache/fory/reflect/FieldAccessor.java b/java/fory-core/src/main/java25/org/apache/fory/reflect/FieldAccessorStrategy.java similarity index 51% rename from java/fory-core/src/main/java25/org/apache/fory/reflect/FieldAccessor.java rename to java/fory-core/src/main/java25/org/apache/fory/reflect/FieldAccessorStrategy.java index 1c06e64634..d56b42a0b3 100644 --- a/java/fory-core/src/main/java25/org/apache/fory/reflect/FieldAccessor.java +++ b/java/fory-core/src/main/java25/org/apache/fory/reflect/FieldAccessorStrategy.java @@ -19,238 +19,21 @@ package org.apache.fory.reflect; -import java.lang.invoke.MethodHandle; import java.lang.invoke.MethodHandles; -import java.lang.invoke.MethodType; import java.lang.invoke.VarHandle; import java.lang.reflect.Field; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; import java.lang.reflect.Modifier; -import java.util.function.Function; -import java.util.function.Predicate; -import java.util.function.ToDoubleFunction; -import java.util.function.ToIntFunction; -import java.util.function.ToLongFunction; -import org.apache.fory.exception.ForyException; -import org.apache.fory.platform.AndroidSupport; import org.apache.fory.platform.GraalvmSupport; +import org.apache.fory.platform.JdkVersion; import org.apache.fory.platform.internal._JDKAccess; import org.apache.fory.type.TypeUtils; import org.apache.fory.util.Preconditions; -import org.apache.fory.util.function.ToByteFunction; -import org.apache.fory.util.function.ToCharFunction; -import org.apache.fory.util.function.ToFloatFunction; -import org.apache.fory.util.function.ToShortFunction; -import org.apache.fory.util.record.RecordUtils; - -/** Field accessor for primitive types and object types. */ -@SuppressWarnings({"unchecked", "rawtypes"}) -public abstract class FieldAccessor { - private static final int BOOLEAN_ACCESS = 1; - private static final int BYTE_ACCESS = 2; - private static final int CHAR_ACCESS = 3; - private static final int SHORT_ACCESS = 4; - private static final int INT_ACCESS = 5; - private static final int LONG_ACCESS = 6; - private static final int FLOAT_ACCESS = 7; - private static final int DOUBLE_ACCESS = 8; - private static final int OBJECT_ACCESS = 9; - - protected final Field field; - // Kept to preserve the root FieldAccessor shape for already compiled internal subclasses. - // JDK25 access is field-owned through VarHandle/MethodHandle and intentionally never uses it. - protected final long fieldOffset; - private final int accessKind; - - public FieldAccessor(Field field) { - this(field, -1); - } - - protected FieldAccessor(Field field, long fieldOffset) { - this.field = field; - this.fieldOffset = fieldOffset; - Preconditions.checkNotNull(field); - this.accessKind = accessKind(field); - } - - private static int accessKind(Field field) { - Class fieldType = field.getType(); - if (fieldType == boolean.class) { - return BOOLEAN_ACCESS; - } else if (fieldType == byte.class) { - return BYTE_ACCESS; - } else if (fieldType == char.class) { - return CHAR_ACCESS; - } else if (fieldType == short.class) { - return SHORT_ACCESS; - } else if (fieldType == int.class) { - return INT_ACCESS; - } else if (fieldType == long.class) { - return LONG_ACCESS; - } else if (fieldType == float.class) { - return FLOAT_ACCESS; - } else if (fieldType == double.class) { - return DOUBLE_ACCESS; - } - return OBJECT_ACCESS; - } - - public abstract Object get(Object obj); - - public void set(Object obj, Object value) { - throw new UnsupportedOperationException("Unsupported for field " + field); - } - - public final void copy(Object sourceObject, Object targetObject) { - switch (accessKind) { - case BOOLEAN_ACCESS: - putBoolean(targetObject, getBoolean(sourceObject)); - return; - case BYTE_ACCESS: - putByte(targetObject, getByte(sourceObject)); - return; - case CHAR_ACCESS: - putChar(targetObject, getChar(sourceObject)); - return; - case SHORT_ACCESS: - putShort(targetObject, getShort(sourceObject)); - return; - case INT_ACCESS: - putInt(targetObject, getInt(sourceObject)); - return; - case LONG_ACCESS: - putLong(targetObject, getLong(sourceObject)); - return; - case FLOAT_ACCESS: - putFloat(targetObject, getFloat(sourceObject)); - return; - case DOUBLE_ACCESS: - putDouble(targetObject, getDouble(sourceObject)); - return; - default: - putObject(targetObject, getObject(sourceObject)); - } - } - - public final void copyObject(Object sourceObject, Object targetObject) { - putObject(targetObject, getObject(sourceObject)); - } - public Field getField() { - return field; - } - - public boolean getBoolean(Object targetObject) { - return (Boolean) get(targetObject); - } - - public void putBoolean(Object targetObject, boolean value) { - set(targetObject, value); - } - - public byte getByte(Object targetObject) { - return (Byte) get(targetObject); - } - - public void putByte(Object targetObject, byte value) { - set(targetObject, value); - } - - public char getChar(Object targetObject) { - return (Character) get(targetObject); - } - - public void putChar(Object targetObject, char value) { - set(targetObject, value); - } - - public short getShort(Object targetObject) { - return (Short) get(targetObject); - } - - public void putShort(Object targetObject, short value) { - set(targetObject, value); - } - - public int getInt(Object targetObject) { - return (Integer) get(targetObject); - } - - public void putInt(Object targetObject, int value) { - set(targetObject, value); - } +final class FieldAccessorStrategy { + private FieldAccessorStrategy() {} - public long getLong(Object targetObject) { - return (Long) get(targetObject); - } - - public void putLong(Object targetObject, long value) { - set(targetObject, value); - } - - public float getFloat(Object targetObject) { - return (Float) get(targetObject); - } - - public void putFloat(Object targetObject, float value) { - set(targetObject, value); - } - - public double getDouble(Object targetObject) { - return (Double) get(targetObject); - } - - public void putDouble(Object targetObject, double value) { - set(targetObject, value); - } - - public final void putObject(Object targetObject, Object object) { - set(targetObject, object); - } - - public final Object getObject(Object targetObject) { - return get(targetObject); - } - - void checkObj(Object obj) { - if (!this.field.getDeclaringClass().isAssignableFrom(obj.getClass())) { - throw new IllegalArgumentException("Illegal class " + obj.getClass()); - } - } - - @Override - public String toString() { - return field.toString(); - } - - public abstract static class FieldGetter extends FieldAccessor { - - private final Object getter; - - protected FieldGetter(Field field, Object getter) { - super(field, -1); - this.getter = getter; - } - - public Object getGetter() { - return getter; - } - } - - public static FieldAccessor createAccessor(Field field) { + static FieldAccessor createAccessor(Field field) { Preconditions.checkArgument(!Modifier.isStatic(field.getModifiers()), field); - if (RecordUtils.isRecord(field.getDeclaringClass())) { - if (AndroidSupport.IS_ANDROID || GraalvmSupport.IN_GRAALVM_NATIVE_IMAGE) { - return new ReflectiveRecordFieldAccessor(field); - } - return createRecordAccessor(field); - } - if (AndroidSupport.IS_ANDROID) { - // Android field access must stay reflection-owned: no Unsafe offsets, trusted lookups, - // generated accessors, or primitive-specific reflection subclasses. - return new ReflectionFieldAccessor(field); - } if (GraalvmSupport.isGraalBuildTime() || GraalvmSupport.IN_GRAALVM_NATIVE_IMAGE) { return new GeneratedAccessor(field); } @@ -261,12 +44,8 @@ public static FieldAccessor createAccessor(Field field) { return createVarHandleAccessor(field); } - public static FieldAccessor createStaticAccessor(Field field) { + static FieldAccessor createStaticAccessor(Field field) { Preconditions.checkArgument(Modifier.isStatic(field.getModifiers()), field); - if (AndroidSupport.IS_ANDROID) { - field.setAccessible(true); - return new ReflectiveStaticFieldAccessor(field); - } return createVarHandleAccessor(field); } @@ -292,37 +71,14 @@ private static FieldAccessor createVarHandleAccessor(Field field) { } } - private static FieldAccessor createRecordAccessor(Field field) { - MethodHandle getter = recordGetter(field); - if (field.getType() == boolean.class) { - return new BooleanGetter(field, getter); - } else if (field.getType() == byte.class) { - return new ByteGetter(field, getter); - } else if (field.getType() == char.class) { - return new CharGetter(field, getter); - } else if (field.getType() == short.class) { - return new ShortGetter(field, getter); - } else if (field.getType() == int.class) { - return new IntGetter(field, getter); - } else if (field.getType() == long.class) { - return new LongGetter(field, getter); - } else if (field.getType() == float.class) { - return new FloatGetter(field, getter); - } else if (field.getType() == double.class) { - return new DoubleGetter(field, getter); - } else { - return new ObjectGetter(field, getter); - } - } - private static VarHandle fieldHandle(Field field) { try { if (canUsePublicField(field)) { try { return findFieldHandle(MethodHandles.publicLookup(), field); } catch (IllegalAccessException ignored) { - // The package may be opened but not exported. Fall through to privateLookupIn so - // --add-opens still enables access for named-module users. + // Fall through to the JDK25 trusted lookup. It is enabled by opening + // java.base/java.lang.invoke to Fory, not by opening every target package. } } return findFieldHandle(privateLookup(field), field); @@ -333,23 +89,6 @@ private static VarHandle fieldHandle(Field field) { } } - private static MethodHandle recordGetter(Field field) { - try { - if (Modifier.isPublic(field.getDeclaringClass().getModifiers())) { - try { - return findRecordGetter(MethodHandles.publicLookup(), field); - } catch (IllegalAccessException ignored) { - // The package may be opened but not exported. Fall through to privateLookupIn. - } - } - return findRecordGetter(privateLookup(field), field); - } catch (IllegalAccessException e) { - throw accessFailure(field, e); - } catch (NoSuchMethodException e) { - throw new IllegalStateException("Failed to find record accessor for field " + field, e); - } - } - private static VarHandle findFieldHandle(MethodHandles.Lookup lookup, Field field) throws IllegalAccessException, NoSuchFieldException { if (Modifier.isStatic(field.getModifiers())) { @@ -359,12 +98,6 @@ private static VarHandle findFieldHandle(MethodHandles.Lookup lookup, Field fiel return lookup.findVarHandle(field.getDeclaringClass(), field.getName(), field.getType()); } - private static MethodHandle findRecordGetter(MethodHandles.Lookup lookup, Field field) - throws IllegalAccessException, NoSuchMethodException { - return lookup.findVirtual( - field.getDeclaringClass(), field.getName(), MethodType.methodType(field.getType())); - } - private static MethodHandles.Lookup privateLookup(Field field) { Class declaringClass = field.getDeclaringClass(); return _JDKAccess.privateLookupIn(declaringClass, MethodHandles.lookup()); @@ -376,207 +109,160 @@ private static boolean canUsePublicField(Field field) { } private static IllegalStateException accessFailure(Field field, Throwable cause) { - Class declaringClass = field.getDeclaringClass(); - Module targetModule = declaringClass.getModule(); - Package targetPackage = declaringClass.getPackage(); - String packageName = targetPackage == null ? "" : targetPackage.getName(); - String openTarget = - moduleName(targetModule) + (packageName.isEmpty() ? "" : "/" + packageName); return new IllegalStateException( "Cannot access field " + field - + " because package " - + packageName - + " in module " - + moduleName(targetModule) - + " is not open to " - + moduleName(FieldAccessor.class.getModule()) - + ". For named modules, open the package with --add-opens=" - + openTarget - + "=org.apache.fory.core,org.apache.fory.format", + + ". JDK25 zero-Unsafe mode requires java.base/java.lang.invoke to be open " + + "to org.apache.fory.core", cause); } - private static String moduleName(Module module) { - String name = module.getName(); - return name == null ? "" : name; - } - private static UnsupportedOperationException unsupportedWrite(Field field, Throwable cause) { return new UnsupportedOperationException( "Field cannot be written through supported JDK access APIs: " + field, cause); } private static IllegalStateException finalMutationFailure(Field field, Throwable cause) { + String versionMessage = + JdkVersion.MAJOR_VERSION >= 26 + ? "On JDK26+, start the JVM with " + + "--enable-final-field-mutation=org.apache.fory.core, or ALL-UNNAMED " + + "when Fory is loaded from the classpath. " + : ""; return new IllegalStateException( "Cannot write final field " + field - + ". On JDK25+, start the JVM with " - + "--enable-final-field-mutation=org.apache.fory.core and open the declaring " - + "package to org.apache.fory.core,org.apache.fory.format.", + + ". " + + versionMessage + + "The declaring package must be open to org.apache.fory.core when it is in a " + + "named module.", cause); } - private static RuntimeException getterFailure(Field field, Throwable cause) { - return new RuntimeException("Failed to read record field: " + field, cause); - } - private static RuntimeException accessorFailure(Field field, Throwable cause) { return new RuntimeException("Failed to access field: " + field, cause); } - static final class ReflectiveRecordFieldAccessor extends FieldGetter { - private final Method accessor; - - ReflectiveRecordFieldAccessor(Field field) { - super(field, null); - try { - accessor = field.getDeclaringClass().getDeclaredMethod(field.getName()); - accessor.setAccessible(true); - } catch (NoSuchMethodException | RuntimeException e) { - throw new ForyException("Failed to create record field accessor for " + field, e); - } - } - - @Override - public Object get(Object obj) { - checkObj(obj); - try { - return accessor.invoke(obj); - } catch (IllegalAccessException | IllegalArgumentException e) { - throw new ForyException("Failed to read record field reflectively: " + field, e); - } catch (InvocationTargetException e) { - throw new ForyException( - "Record accessor threw while reading field: " + field, e.getCause()); - } - } - - @Override - public void set(Object obj, Object value) { - throw new UnsupportedOperationException("Record field is read-only: " + field); - } - } - private abstract static class VarHandleAccessor extends FieldAccessor { protected final VarHandle handle; protected final boolean isStatic; protected final boolean isFinal; - protected volatile MethodHandle finalSetter; + protected volatile Field finalField; VarHandleAccessor(Field field) { - super(field, -1); + super(field); handle = fieldHandle(field); isStatic = Modifier.isStatic(field.getModifiers()); isFinal = Modifier.isFinal(field.getModifiers()); } - private static MethodHandle createFinalSetter(Field field) { + private static Field createFinalField(Field field) { try { field.setAccessible(true); - return privateLookup(field).unreflectSetter(field); - } catch (IllegalAccessException | RuntimeException e) { + return field; + } catch (RuntimeException e) { throw finalMutationFailure(field, e); } } - private MethodHandle finalSetter(Throwable cause) { + private Field finalField(Throwable cause) { if (isStatic || !isFinal) { throw unsupportedWrite(field, cause); } - MethodHandle setter = finalSetter; + Field setter = finalField; if (setter == null) { - setter = createFinalSetter(field); - finalSetter = setter; + setter = createFinalField(field); + finalField = setter; } return setter; } protected void setFinal(Object obj, Object value, Throwable cause) { - MethodHandle setter = finalSetter(cause); + Field setter = finalField(cause); checkObj(obj); try { - setter.invoke(obj, value); - } catch (Throwable e) { + setter.set(obj, value); + } catch (IllegalAccessException | IllegalArgumentException e) { throw finalMutationFailure(field, e); } } protected void setFinalBoolean(Object obj, boolean value, Throwable cause) { - MethodHandle setter = finalSetter(cause); + Field setter = finalField(cause); checkObj(obj); try { - setter.invoke(obj, value); - } catch (Throwable e) { + setter.setBoolean(obj, value); + } catch (IllegalAccessException | IllegalArgumentException e) { throw finalMutationFailure(field, e); } } protected void setFinalByte(Object obj, byte value, Throwable cause) { - MethodHandle setter = finalSetter(cause); + Field setter = finalField(cause); checkObj(obj); try { - setter.invoke(obj, value); - } catch (Throwable e) { + setter.setByte(obj, value); + } catch (IllegalAccessException | IllegalArgumentException e) { throw finalMutationFailure(field, e); } } protected void setFinalChar(Object obj, char value, Throwable cause) { - MethodHandle setter = finalSetter(cause); + Field setter = finalField(cause); checkObj(obj); try { - setter.invoke(obj, value); - } catch (Throwable e) { + setter.setChar(obj, value); + } catch (IllegalAccessException | IllegalArgumentException e) { throw finalMutationFailure(field, e); } } protected void setFinalShort(Object obj, short value, Throwable cause) { - MethodHandle setter = finalSetter(cause); + Field setter = finalField(cause); checkObj(obj); try { - setter.invoke(obj, value); - } catch (Throwable e) { + setter.setShort(obj, value); + } catch (IllegalAccessException | IllegalArgumentException e) { throw finalMutationFailure(field, e); } } protected void setFinalInt(Object obj, int value, Throwable cause) { - MethodHandle setter = finalSetter(cause); + Field setter = finalField(cause); checkObj(obj); try { - setter.invoke(obj, value); - } catch (Throwable e) { + setter.setInt(obj, value); + } catch (IllegalAccessException | IllegalArgumentException e) { throw finalMutationFailure(field, e); } } protected void setFinalLong(Object obj, long value, Throwable cause) { - MethodHandle setter = finalSetter(cause); + Field setter = finalField(cause); checkObj(obj); try { - setter.invoke(obj, value); - } catch (Throwable e) { + setter.setLong(obj, value); + } catch (IllegalAccessException | IllegalArgumentException e) { throw finalMutationFailure(field, e); } } protected void setFinalFloat(Object obj, float value, Throwable cause) { - MethodHandle setter = finalSetter(cause); + Field setter = finalField(cause); checkObj(obj); try { - setter.invoke(obj, value); - } catch (Throwable e) { + setter.setFloat(obj, value); + } catch (IllegalAccessException | IllegalArgumentException e) { throw finalMutationFailure(field, e); } } protected void setFinalDouble(Object obj, double value, Throwable cause) { - MethodHandle setter = finalSetter(cause); + Field setter = finalField(cause); checkObj(obj); try { - setter.invoke(obj, value); - } catch (Throwable e) { + setter.setDouble(obj, value); + } catch (IllegalAccessException | IllegalArgumentException e) { throw finalMutationFailure(field, e); } } @@ -627,43 +313,6 @@ public void putBoolean(Object obj, boolean value) { } } - public static class BooleanGetter extends FieldGetter { - private final Predicate getter; - private final MethodHandle getterHandle; - - public BooleanGetter(Field field, Predicate getter) { - super(field, getter); - this.getter = getter; - getterHandle = null; - Preconditions.checkArgument(field.getType() == boolean.class); - } - - private BooleanGetter(Field field, MethodHandle getter) { - super(field, getter); - this.getter = null; - getterHandle = getter; - Preconditions.checkArgument(field.getType() == boolean.class); - } - - @Override - public Boolean get(Object obj) { - return getBoolean(obj); - } - - @Override - public boolean getBoolean(Object obj) { - checkObj(obj); - if (getterHandle == null) { - return getter.test(obj); - } - try { - return (boolean) getterHandle.invoke(obj); - } catch (Throwable e) { - throw getterFailure(field, e); - } - } - } - /** Primitive byte accessor. */ public static class ByteAccessor extends VarHandleAccessor { public ByteAccessor(Field field) { @@ -709,44 +358,6 @@ public void putByte(Object obj, byte value) { } } - public static class ByteGetter extends FieldGetter { - - private final ToByteFunction getter; - private final MethodHandle getterHandle; - - public ByteGetter(Field field, ToByteFunction getter) { - super(field, getter); - this.getter = getter; - getterHandle = null; - Preconditions.checkArgument(field.getType() == byte.class); - } - - private ByteGetter(Field field, MethodHandle getter) { - super(field, getter); - this.getter = null; - getterHandle = getter; - Preconditions.checkArgument(field.getType() == byte.class); - } - - @Override - public Byte get(Object obj) { - return getByte(obj); - } - - @Override - public byte getByte(Object obj) { - checkObj(obj); - if (getterHandle == null) { - return getter.applyAsByte(obj); - } - try { - return (byte) getterHandle.invoke(obj); - } catch (Throwable e) { - throw getterFailure(field, e); - } - } - } - /** Primitive char accessor. */ public static class CharAccessor extends VarHandleAccessor { public CharAccessor(Field field) { @@ -792,43 +403,6 @@ public void putChar(Object obj, char value) { } } - public static class CharGetter extends FieldGetter { - private final ToCharFunction getter; - private final MethodHandle getterHandle; - - public CharGetter(Field field, ToCharFunction getter) { - super(field, getter); - this.getter = getter; - getterHandle = null; - Preconditions.checkArgument(field.getType() == char.class); - } - - private CharGetter(Field field, MethodHandle getter) { - super(field, getter); - this.getter = null; - getterHandle = getter; - Preconditions.checkArgument(field.getType() == char.class); - } - - @Override - public Character get(Object obj) { - return getChar(obj); - } - - @Override - public char getChar(Object obj) { - checkObj(obj); - if (getterHandle == null) { - return getter.applyAsChar(obj); - } - try { - return (char) getterHandle.invoke(obj); - } catch (Throwable e) { - throw getterFailure(field, e); - } - } - } - /** Primitive short accessor. */ public static class ShortAccessor extends VarHandleAccessor { public ShortAccessor(Field field) { @@ -874,43 +448,6 @@ public void putShort(Object obj, short value) { } } - public static class ShortGetter extends FieldGetter { - private final ToShortFunction getter; - private final MethodHandle getterHandle; - - public ShortGetter(Field field, ToShortFunction getter) { - super(field, getter); - this.getter = getter; - getterHandle = null; - Preconditions.checkArgument(field.getType() == short.class); - } - - private ShortGetter(Field field, MethodHandle getter) { - super(field, getter); - this.getter = null; - getterHandle = getter; - Preconditions.checkArgument(field.getType() == short.class); - } - - @Override - public Short get(Object obj) { - return getShort(obj); - } - - @Override - public short getShort(Object obj) { - checkObj(obj); - if (getterHandle == null) { - return getter.applyAsShort(obj); - } - try { - return (short) getterHandle.invoke(obj); - } catch (Throwable e) { - throw getterFailure(field, e); - } - } - } - /** Primitive int accessor. */ public static class IntAccessor extends VarHandleAccessor { public IntAccessor(Field field) { @@ -956,43 +493,6 @@ public void putInt(Object obj, int value) { } } - public static class IntGetter extends FieldGetter { - private final ToIntFunction getter; - private final MethodHandle getterHandle; - - public IntGetter(Field field, ToIntFunction getter) { - super(field, getter); - this.getter = getter; - getterHandle = null; - Preconditions.checkArgument(field.getType() == int.class); - } - - private IntGetter(Field field, MethodHandle getter) { - super(field, getter); - this.getter = null; - getterHandle = getter; - Preconditions.checkArgument(field.getType() == int.class); - } - - @Override - public Integer get(Object obj) { - return getInt(obj); - } - - @Override - public int getInt(Object obj) { - checkObj(obj); - if (getterHandle == null) { - return getter.applyAsInt(obj); - } - try { - return (int) getterHandle.invoke(obj); - } catch (Throwable e) { - throw getterFailure(field, e); - } - } - } - /** Primitive long accessor. */ public static class LongAccessor extends VarHandleAccessor { public LongAccessor(Field field) { @@ -1038,43 +538,6 @@ public void putLong(Object obj, long value) { } } - public static class LongGetter extends FieldGetter { - private final ToLongFunction getter; - private final MethodHandle getterHandle; - - public LongGetter(Field field, ToLongFunction getter) { - super(field, getter); - this.getter = getter; - getterHandle = null; - Preconditions.checkArgument(field.getType() == long.class); - } - - private LongGetter(Field field, MethodHandle getter) { - super(field, getter); - this.getter = null; - getterHandle = getter; - Preconditions.checkArgument(field.getType() == long.class); - } - - @Override - public Long get(Object obj) { - return getLong(obj); - } - - @Override - public long getLong(Object obj) { - checkObj(obj); - if (getterHandle == null) { - return getter.applyAsLong(obj); - } - try { - return (long) getterHandle.invoke(obj); - } catch (Throwable e) { - throw getterFailure(field, e); - } - } - } - /** Primitive float accessor. */ public static class FloatAccessor extends VarHandleAccessor { public FloatAccessor(Field field) { @@ -1120,43 +583,6 @@ public void putFloat(Object obj, float value) { } } - public static class FloatGetter extends FieldGetter { - private final ToFloatFunction getter; - private final MethodHandle getterHandle; - - public FloatGetter(Field field, ToFloatFunction getter) { - super(field, getter); - this.getter = getter; - getterHandle = null; - Preconditions.checkArgument(field.getType() == float.class); - } - - private FloatGetter(Field field, MethodHandle getter) { - super(field, getter); - this.getter = null; - getterHandle = getter; - Preconditions.checkArgument(field.getType() == float.class); - } - - @Override - public Float get(Object obj) { - return getFloat(obj); - } - - @Override - public float getFloat(Object obj) { - checkObj(obj); - if (getterHandle == null) { - return getter.applyAsFloat(obj); - } - try { - return (float) getterHandle.invoke(obj); - } catch (Throwable e) { - throw getterFailure(field, e); - } - } - } - /** Primitive double accessor. */ public static class DoubleAccessor extends VarHandleAccessor { public DoubleAccessor(Field field) { @@ -1202,43 +628,6 @@ public void putDouble(Object obj, double value) { } } - public static class DoubleGetter extends FieldGetter { - private final ToDoubleFunction getter; - private final MethodHandle getterHandle; - - public DoubleGetter(Field field, ToDoubleFunction getter) { - super(field, getter); - this.getter = getter; - getterHandle = null; - Preconditions.checkArgument(field.getType() == double.class); - } - - private DoubleGetter(Field field, MethodHandle getter) { - super(field, getter); - this.getter = null; - getterHandle = getter; - Preconditions.checkArgument(field.getType() == double.class); - } - - @Override - public Double get(Object obj) { - return getDouble(obj); - } - - @Override - public double getDouble(Object obj) { - checkObj(obj); - if (getterHandle == null) { - return getter.applyAsDouble(obj); - } - try { - return (double) getterHandle.invoke(obj); - } catch (Throwable e) { - throw getterFailure(field, e); - } - } - } - /** Object accessor. */ public static class ObjectAccessor extends VarHandleAccessor { public ObjectAccessor(Field field) { @@ -1274,62 +663,6 @@ public void set(Object obj, Object value) { } } - public static class ObjectGetter extends FieldGetter { - private final Function getter; - private final MethodHandle getterHandle; - - public ObjectGetter(Field field, Function getter) { - super(field, getter); - this.getter = getter; - getterHandle = null; - Preconditions.checkArgument(!field.getType().isPrimitive(), field); - } - - private ObjectGetter(Field field, MethodHandle getter) { - super(field, getter); - this.getter = null; - getterHandle = getter; - Preconditions.checkArgument(!field.getType().isPrimitive(), field); - } - - @Override - public Object get(Object obj) { - checkObj(obj); - if (getterHandle == null) { - return getter.apply(obj); - } - try { - return getterHandle.invoke(obj); - } catch (Throwable e) { - throw getterFailure(field, e); - } - } - } - - static final class ReflectiveStaticFieldAccessor extends FieldAccessor { - ReflectiveStaticFieldAccessor(Field field) { - super(field, -1); - } - - @Override - public Object get(Object obj) { - try { - return field.get(null); - } catch (IllegalAccessException | IllegalArgumentException e) { - throw new ForyException("Failed to read static field reflectively: " + field, e); - } - } - - @Override - public void set(Object obj, Object value) { - try { - field.set(null, value); - } catch (IllegalAccessException | IllegalArgumentException e) { - throw new ForyException("Failed to write static field reflectively: " + field, e); - } - } - } - static final class StaticObjectAccessor extends ObjectAccessor { StaticObjectAccessor(Field field) { super(field); diff --git a/java/fory-core/src/main/java25/org/apache/fory/reflect/UnsafeObjectAllocator.java b/java/fory-core/src/main/java25/org/apache/fory/reflect/UnsafeObjectAllocator.java new file mode 100644 index 0000000000..274e711ad6 --- /dev/null +++ b/java/fory-core/src/main/java25/org/apache/fory/reflect/UnsafeObjectAllocator.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.reflect; + +import java.io.ObjectStreamClass; +import java.io.Serializable; +import java.lang.invoke.MethodHandle; +import java.lang.invoke.MethodType; +import java.lang.reflect.InvocationTargetException; +import org.apache.fory.annotation.Internal; +import org.apache.fory.exception.ForyException; +import org.apache.fory.platform.internal._JDKAccess; +import org.apache.fory.util.ExceptionUtils; + +/** JDK25 replacement for the JDK8-24 Unsafe allocator. */ +@Internal +final class UnsafeObjectAllocator { + private UnsafeObjectAllocator() {} + + static T allocate(Class type) { + if (Serializable.class.isAssignableFrom(type)) { + try { + return type.cast(ObjectStreamClassAccess.newInstance(type)); + } catch (UnsupportedOperationException e) { + throw unsupported(type, e); + } catch (InstantiationException e) { + throw unsupported(type, e); + } catch (InvocationTargetException e) { + throw ExceptionUtils.throwException(e.getTargetException()); + } catch (Throwable e) { + throw new ForyException("Failed to create an instance for " + type, e); + } + } + throw unsupported(type, null); + } + + private static ForyException unsupported(Class type, Throwable cause) { + return new ForyException( + "Cannot create a constructor-bypassing instance for " + + type + + " in JDK25+ zero-Unsafe mode. Provide an accessible no-arg constructor, " + + "annotate a constructor with @ForyConstructor, register a constructor with " + + "BaseFory.registerConstructor, use a record canonical constructor, or register a " + + "custom serializer.", + cause); + } + + private static final class ObjectStreamClassAccess { + private static final MethodHandle NEW_INSTANCE = newInstanceHandle(); + + private static MethodHandle newInstanceHandle() { + try { + return _JDKAccess + ._trustedLookup(ObjectStreamClass.class) + .findVirtual( + ObjectStreamClass.class, "newInstance", MethodType.methodType(Object.class)); + } catch (ReflectiveOperationException e) { + throw new ForyException( + "JDK25+ Serializable object creation requires java.base/java.lang.invoke to be open " + + "to org.apache.fory.core", + e); + } + } + + private static Object newInstance(Class type) throws Throwable { + return NEW_INSTANCE.invoke(ObjectStreamClass.lookupAny(type)); + } + } +} diff --git a/java/fory-core/src/test/java/org/apache/fory/ForyTest.java b/java/fory-core/src/test/java/org/apache/fory/ForyTest.java index 12684e9326..ce93278584 100644 --- a/java/fory-core/src/test/java/org/apache/fory/ForyTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/ForyTest.java @@ -27,7 +27,6 @@ import com.google.common.collect.HashBasedTable; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import java.beans.ConstructorProperties; import java.io.Serializable; import java.lang.invoke.MethodHandles; import java.math.BigDecimal; @@ -54,6 +53,7 @@ import lombok.Data; import lombok.EqualsAndHashCode; import org.apache.fory.annotation.Expose; +import org.apache.fory.annotation.ForyConstructor; import org.apache.fory.annotation.Ignore; import org.apache.fory.builder.Generated; import org.apache.fory.config.ForyBuilder; @@ -439,14 +439,13 @@ private static class IgnoreFields { @Ignore long f2; long f3; - @ConstructorProperties({"f1", "f2", "f3"}) IgnoreFields(int f1, long f2, long f3) { this.f1 = f1; this.f2 = f2; this.f3 = f3; } - @ConstructorProperties({"f3"}) + @ForyConstructor({"f3"}) IgnoreFields(long f3) { this.f3 = f3; } @@ -469,7 +468,6 @@ private static class ExposeFields { @Expose ImmutableMap map1; ImmutableMap map2; - @ConstructorProperties({"f1", "f2", "f3", "map1", "map2"}) ExposeFields( int f1, long f2, @@ -483,7 +481,7 @@ private static class ExposeFields { this.map2 = map2; } - @ConstructorProperties({"f1", "f2", "map1"}) + @ForyConstructor({"f1", "f2", "map1"}) ExposeFields(int f1, long f2, ImmutableMap map1) { this.f1 = f1; this.f2 = f2; @@ -510,7 +508,7 @@ private static class ExposeFields2 { @Ignore long f2; long f3; - @ConstructorProperties({"f1", "f2", "f3"}) + @ForyConstructor({"f1", "f2", "f3"}) ExposeFields2(int f1, long f2, long f3) { this.f1 = f1; this.f2 = f2; @@ -711,7 +709,7 @@ static class Struct1 { int f1; String f2; - @ConstructorProperties({"f1", "f2"}) + @ForyConstructor({"f1", "f2"}) public Struct1(int f1, String f2) { this.f1 = f1; this.f2 = f2; @@ -774,7 +772,7 @@ static class MaxDepth { int f1; Object f2; - @ConstructorProperties({"f1", "f2"}) + @ForyConstructor({"f1", "f2"}) MaxDepth(int f1, Object f2) { this.f1 = f1; this.f2 = f2; diff --git a/java/fory-core/src/test/java/org/apache/fory/TestUtils.java b/java/fory-core/src/test/java/org/apache/fory/TestUtils.java index 22bd3edce5..6fa564d930 100644 --- a/java/fory-core/src/test/java/org/apache/fory/TestUtils.java +++ b/java/fory-core/src/test/java/org/apache/fory/TestUtils.java @@ -28,7 +28,17 @@ import java.lang.management.ManagementFactory; import java.lang.reflect.Field; import java.lang.reflect.Modifier; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; import java.util.function.Supplier; import java.util.stream.Collectors; import org.apache.fory.collection.Tuple3; @@ -44,7 +54,7 @@ /** Test utils. */ public class TestUtils { public static List javaCommand(Class mainClass) { - return javaCommand(System.getProperty("java.class.path"), mainClass); + return javaCommand(forkClassPath(), mainClass); } public static List javaCommand( @@ -53,6 +63,15 @@ public static List javaCommand( command.add(System.getProperty("java.home") + File.separator + "bin" + File.separator + "java"); command.addAll(forkJvmArgs()); Collections.addAll(command, extraJvmArgs); + if (JdkVersion.MAJOR_VERSION >= 25) { + String modulePath = System.getProperty("jdk.module.path"); + if (modulePath != null && !modulePath.isEmpty()) { + command.add("--module-path"); + command.add(modulePath); + command.add("--add-modules"); + command.add("org.apache.fory.core"); + } + } command.add("-cp"); command.add(classPath); command.add(mainClass.getName()); @@ -62,23 +81,39 @@ public static List javaCommand( private static List forkJvmArgs() { List args = new ArrayList<>(); if (JdkVersion.MAJOR_VERSION >= 25) { - args.add("--add-opens=java.base/java.lang=ALL-UNNAMED"); args.add("--add-opens=java.base/java.lang.invoke=ALL-UNNAMED"); - args.add("--add-opens=java.base/java.lang.reflect=ALL-UNNAMED"); - args.add("--add-opens=java.base/jdk.internal.reflect=ALL-UNNAMED"); - args.add("--add-opens=java.base/java.util=ALL-UNNAMED"); - args.add("--add-opens=java.base/java.util.concurrent=ALL-UNNAMED"); if (hasInputArg("--sun-misc-unsafe-memory-access=deny")) { args.add("--sun-misc-unsafe-memory-access=deny"); } + finalFieldMutationArg().ifPresent(args::add); } return args; } + public static String forkClassPath() { + LinkedHashSet elements = new LinkedHashSet<>(); + addPathElements(elements, System.getProperty("surefire.real.class.path")); + addPathElements(elements, System.getProperty("java.class.path")); + return String.join(File.pathSeparator, elements); + } + + private static void addPathElements(Set elements, String path) { + if (path == null || path.isEmpty()) { + return; + } + Collections.addAll(elements, path.split(java.util.regex.Pattern.quote(File.pathSeparator))); + } + private static boolean hasInputArg(String arg) { return ManagementFactory.getRuntimeMXBean().getInputArguments().contains(arg); } + private static Optional finalFieldMutationArg() { + return ManagementFactory.getRuntimeMXBean().getInputArguments().stream() + .filter(arg -> arg.startsWith("--enable-final-field-mutation=")) + .findFirst(); + } + @SuppressWarnings("unchecked") public static T getFieldValue(Object obj, String fieldName) { return (T) diff --git a/java/fory-core/src/test/java/org/apache/fory/builder/BuilderUnsafeClassGraphTest.java b/java/fory-core/src/test/java/org/apache/fory/builder/BuilderUnsafeClassGraphTest.java new file mode 100644 index 0000000000..9a76cf607a --- /dev/null +++ b/java/fory-core/src/test/java/org/apache/fory/builder/BuilderUnsafeClassGraphTest.java @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.builder; + +import static org.testng.Assert.assertTrue; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Pattern; +import java.util.stream.Stream; +import org.testng.annotations.Test; + +public class BuilderUnsafeClassGraphTest { + private static final Path ROOT_BUILDER = Paths.get("src/main/java/org/apache/fory/builder"); + private static final Path JAVA25_BUILDER = Paths.get("src/main/java25/org/apache/fory/builder"); + private static final Pattern ROOT_UNSAFE_REFERENCE = + Pattern.compile( + "import\\s+sun\\.misc\\.Unsafe|" + + "sun\\.misc\\.Unsafe|" + + "\\bUnsafe\\.class\\b|" + + "_JDKAccess\\.UNSAFE|" + + "TypeRef\\.of\\(Unsafe"); + private static final Pattern JAVA25_UNSAFE_REFERENCE = + Pattern.compile( + "import\\s+sun\\.misc\\.Unsafe|" + + "sun\\.misc\\.Unsafe|" + + "\\bUnsafe\\.class\\b|" + + "_JDKAccess\\.UNSAFE|" + + "TypeRef\\.of\\(Unsafe|" + + "Class\\.forName\\(\"sun\\.misc\\.Unsafe\"\\)"); + + @Test + public void testUnsafeOwner() throws IOException { + List violations = new ArrayList<>(); + try (Stream paths = Files.walk(ROOT_BUILDER)) { + paths + .filter(path -> path.toString().endsWith(".java")) + .forEach( + path -> { + try { + String source = new String(Files.readAllBytes(path), StandardCharsets.UTF_8); + if (!ROOT_UNSAFE_REFERENCE.matcher(source).find()) { + return; + } + Path relative = ROOT_BUILDER.relativize(path); + Path replacement = JAVA25_BUILDER.resolve(relative); + if (!Files.exists(replacement)) { + violations.add(relative.toString().replace('\\', '/')); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + } + assertTrue( + violations.isEmpty(), + "Root builder classes that mention Unsafe must have Java 25 replacements: " + violations); + } + + @Test + public void testJava25OwnerIsClean() throws IOException { + List violations = new ArrayList<>(); + try (Stream paths = Files.walk(JAVA25_BUILDER)) { + paths + .filter(path -> path.toString().endsWith(".java")) + .forEach( + path -> { + try { + String source = new String(Files.readAllBytes(path), StandardCharsets.UTF_8); + if (JAVA25_UNSAFE_REFERENCE.matcher(source).find()) { + violations.add(JAVA25_BUILDER.relativize(path).toString().replace('\\', '/')); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + } + assertTrue( + violations.isEmpty(), + "Java 25 builder replacements must not reference sun.misc.Unsafe: " + violations); + } +} diff --git a/java/fory-core/src/test/java/org/apache/fory/codegen/pkgprivate/PackagePrivateMapKeyTest.java b/java/fory-core/src/test/java/org/apache/fory/codegen/pkgprivate/PackagePrivateMapKeyTest.java index e13bbff62c..0f9076cf06 100644 --- a/java/fory-core/src/test/java/org/apache/fory/codegen/pkgprivate/PackagePrivateMapKeyTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/codegen/pkgprivate/PackagePrivateMapKeyTest.java @@ -21,7 +21,6 @@ import static org.testng.Assert.assertEquals; -import java.beans.ConstructorProperties; import java.io.Serializable; import java.util.EnumMap; import java.util.HashMap; @@ -30,6 +29,7 @@ import java.util.Set; import org.apache.fory.Fory; import org.apache.fory.ThreadSafeFory; +import org.apache.fory.annotation.ForyConstructor; import org.testng.annotations.Test; /** Regression test for codegen CompileException when map key/value types are package-private. */ @@ -70,7 +70,7 @@ class ReproNode implements Serializable { Set children; Map> parents; - @ConstructorProperties({"type", "id"}) + @ForyConstructor({"type", "id"}) ReproNode(ReproType type, String id) { this(type, id, new HashSet<>(), new EnumMap<>(ReproType.class)); } @@ -92,7 +92,7 @@ class ReproContainer implements Serializable { this(new EnumMap<>(ReproType.class), version); } - @ConstructorProperties({"nodes", "version"}) + @ForyConstructor({"nodes", "version"}) ReproContainer(Map> nodes, String version) { this.nodes = nodes; this.version = version; diff --git a/java/fory-core/src/test/java/org/apache/fory/reflect/FieldAccessorTest.java b/java/fory-core/src/test/java/org/apache/fory/reflect/FieldAccessorTest.java index 0b333ec972..d20309726c 100644 --- a/java/fory-core/src/test/java/org/apache/fory/reflect/FieldAccessorTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/reflect/FieldAccessorTest.java @@ -20,15 +20,15 @@ package org.apache.fory.reflect; import java.io.ByteArrayOutputStream; -import java.io.File; import java.io.IOException; import java.io.InputStream; import java.lang.reflect.Field; import java.nio.charset.StandardCharsets; import lombok.AllArgsConstructor; +import org.apache.fory.TestUtils; import org.apache.fory.platform.AndroidSupport; import org.apache.fory.platform.JdkVersion; -import org.apache.fory.reflect.FieldAccessor.GeneratedAccessor; +import org.apache.fory.reflect.FieldAccessorStrategy.GeneratedAccessor; import org.testng.Assert; import org.testng.annotations.Test; @@ -110,14 +110,8 @@ private static boolean isHidden(Class cls) throws Exception { @Test public void testAndroidReflectionFieldAccessorPaths() throws Exception { - String javaBin = - System.getProperty("java.home") + File.separator + "bin" + File.separator + "java"; Process process = - new ProcessBuilder( - javaBin, - "-cp", - System.getProperty("java.class.path"), - AndroidReflectionFieldAccessorProbe.class.getName()) + new ProcessBuilder(TestUtils.javaCommand(AndroidReflectionFieldAccessorProbe.class)) .redirectErrorStream(true) .start(); String output = readFully(process.getInputStream()); diff --git a/java/fory-core/src/test/java/org/apache/fory/reflect/ObjectCreatorsTest.java b/java/fory-core/src/test/java/org/apache/fory/reflect/ObjectCreatorsTest.java index 8862cf421c..bda955b4c4 100644 --- a/java/fory-core/src/test/java/org/apache/fory/reflect/ObjectCreatorsTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/reflect/ObjectCreatorsTest.java @@ -20,13 +20,14 @@ package org.apache.fory.reflect; import java.io.ByteArrayOutputStream; -import java.io.File; import java.io.IOException; import java.io.InputStream; import java.nio.charset.StandardCharsets; import java.util.concurrent.ArrayBlockingQueue; +import org.apache.fory.TestUtils; import org.apache.fory.exception.ForyException; import org.apache.fory.platform.AndroidSupport; +import org.apache.fory.platform.JdkVersion; import org.apache.fory.reflect.ObjectCreators.ParentNoArgCtrObjectCreator; import org.testng.Assert; import org.testng.annotations.Test; @@ -44,6 +45,9 @@ public NoCtrTestClass(int f1) { @Test public void testObjectCreator() { + if (JdkVersion.MAJOR_VERSION >= 25) { + return; + } ParentNoArgCtrObjectCreator creator = new ParentNoArgCtrObjectCreator<>(ArrayBlockingQueue.class); Assert.assertEquals(creator.newInstance().getClass(), ArrayBlockingQueue.class); @@ -54,14 +58,8 @@ public void testObjectCreator() { @Test public void testAndroidObjectCreators() throws Exception { - String javaBin = - System.getProperty("java.home") + File.separator + "bin" + File.separator + "java"; Process process = - new ProcessBuilder( - javaBin, - "-cp", - System.getProperty("java.class.path"), - AndroidObjectCreatorProbe.class.getName()) + new ProcessBuilder(TestUtils.javaCommand(AndroidObjectCreatorProbe.class)) .redirectErrorStream(true) .start(); String output = readFully(process.getInputStream()); diff --git a/java/fory-core/src/test/java/org/apache/fory/reflect/ReflectionUtilsTest.java b/java/fory-core/src/test/java/org/apache/fory/reflect/ReflectionUtilsTest.java index 64170f72f7..b00d3e54b7 100644 --- a/java/fory-core/src/test/java/org/apache/fory/reflect/ReflectionUtilsTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/reflect/ReflectionUtilsTest.java @@ -117,9 +117,8 @@ public void testGetNoArgConstructor() throws Throwable { Constructor ctr = ReflectionUtils.getNoArgConstructor(NoArgConstructor1.class); Assert.assertNull(ctr); - // ctr is generated by jdk.internal.reflect.ReflectionFactory.generateConstructor, - // `unreflectConstructor` doesn't work, see ParentNoArgCtrObjectCreator to get parent no-arg - // ctr. + // ReflectionFactory serialization constructors are invoked directly by + // ParentNoArgCtrObjectCreator. // MethodHandle handle = lookup.unreflectConstructor(ctr); // System.out.println(ctr); // System.out.println(handle); diff --git a/java/fory-core/src/test/java/org/apache/fory/serializer/AndroidJvmRoundTripTest.java b/java/fory-core/src/test/java/org/apache/fory/serializer/AndroidJvmRoundTripTest.java index 9b8f2868b5..e526deba4b 100644 --- a/java/fory-core/src/test/java/org/apache/fory/serializer/AndroidJvmRoundTripTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/serializer/AndroidJvmRoundTripTest.java @@ -161,7 +161,6 @@ private static void addAddOpens(ArrayList command) { addAddOpens(command, "java.base/java.lang=ALL-UNNAMED"); addAddOpens(command, "java.base/java.lang.invoke=ALL-UNNAMED"); addAddOpens(command, "java.base/java.lang.reflect=ALL-UNNAMED"); - addAddOpens(command, "java.base/jdk.internal.reflect=ALL-UNNAMED"); addAddOpens(command, "java.base/java.util=ALL-UNNAMED"); } diff --git a/java/fory-core/src/test/java/org/apache/fory/serializer/ArraySerializersTest.java b/java/fory-core/src/test/java/org/apache/fory/serializer/ArraySerializersTest.java index 56d19e4291..891b5fe56a 100644 --- a/java/fory-core/src/test/java/org/apache/fory/serializer/ArraySerializersTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/serializer/ArraySerializersTest.java @@ -23,7 +23,6 @@ import static org.testng.Assert.assertThrows; import static org.testng.Assert.assertTrue; -import java.beans.ConstructorProperties; import java.lang.reflect.Array; import java.lang.reflect.Field; import java.util.Arrays; @@ -36,6 +35,7 @@ import lombok.EqualsAndHashCode; import org.apache.fory.Fory; import org.apache.fory.ForyTestBase; +import org.apache.fory.annotation.ForyConstructor; import org.apache.fory.config.ForyBuilder; import org.apache.fory.config.Int64Encoding; import org.apache.fory.context.MetaReadContext; @@ -487,7 +487,7 @@ public void testArrayStructZeroCopy(Fory fory) { static class A { final int f1; - @ConstructorProperties({"f1"}) + @ForyConstructor({"f1"}) A(int f1) { this.f1 = f1; } @@ -497,7 +497,7 @@ static class A { static class B extends A { final String f2; - @ConstructorProperties({"f1", "f2"}) + @ForyConstructor({"f1", "f2"}) B(int f1, String f2) { super(f1); this.f2 = f2; @@ -521,7 +521,7 @@ public GenericArrayWrapper(Class clazz, int capacity) { this.array = (T[]) Array.newInstance(clazz, capacity); } - @ConstructorProperties({"array"}) + @ForyConstructor({"array"}) public GenericArrayWrapper(T[] array) { this.array = array; } diff --git a/java/fory-core/src/test/java/org/apache/fory/serializer/ObjectSerializerTest.java b/java/fory-core/src/test/java/org/apache/fory/serializer/ObjectSerializerTest.java index f0b2235db3..92f7d91fd4 100644 --- a/java/fory-core/src/test/java/org/apache/fory/serializer/ObjectSerializerTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/serializer/ObjectSerializerTest.java @@ -23,15 +23,16 @@ import static org.testng.Assert.assertNotSame; import static org.testng.Assert.assertSame; -import java.beans.ConstructorProperties; import java.io.ByteArrayOutputStream; -import java.io.File; import java.io.IOException; import java.io.InputStream; +import java.lang.reflect.Constructor; import java.nio.charset.StandardCharsets; import lombok.Data; import org.apache.fory.Fory; import org.apache.fory.ForyTestBase; +import org.apache.fory.TestUtils; +import org.apache.fory.annotation.ForyConstructor; import org.apache.fory.annotation.ForyField; import org.apache.fory.builder.CodecUtils; import org.apache.fory.memory.MemoryBuffer; @@ -181,7 +182,7 @@ public static final class ConstructorCycle { private final String name; private ConstructorCycle next; - @ConstructorProperties("name") + @ForyConstructor("name") public ConstructorCycle(String name) { this.name = name; } @@ -194,7 +195,7 @@ public static final class ConstructorCycleBeforeFinal { @ForyField(id = 1) private final String name; - @ConstructorProperties("name") + @ForyConstructor("name") public ConstructorCycleBeforeFinal(String name) { this.name = name; } @@ -204,7 +205,7 @@ public static final class ConstructorOrder { private int id; private final String name; - @ConstructorProperties("name") + @ForyConstructor("name") public ConstructorOrder(String name) { this.name = name; } @@ -220,7 +221,7 @@ public static final class ConstructorInterveningRef { @ForyField(id = 2) private Object second; - @ConstructorProperties("name") + @ForyConstructor("name") public ConstructorInterveningRef(String name) { this.name = name; } @@ -229,7 +230,7 @@ public ConstructorInterveningRef(String name) { public static final class ConstructorBackrefRoot { private final ConstructorBackrefChild child; - @ConstructorProperties("child") + @ForyConstructor("child") public ConstructorBackrefRoot(ConstructorBackrefChild child) { this.child = child; } @@ -239,6 +240,19 @@ public static final class ConstructorBackrefChild { private ConstructorBackrefRoot root; } + public static final class RegisteredCtorBean { + @ForyField(id = 0) + private final String name; + + @ForyField(id = 1) + private final int age; + + private RegisteredCtorBean(int age, String name) { + this.name = name; + this.age = age; + } + } + public static final class FinalNoArgBean { private final int id; private final String name; @@ -260,7 +274,7 @@ public static final class FinalPostCtorBean { private final int id; private String label; - @ConstructorProperties("label") + @ForyConstructor("label") public FinalPostCtorBean(String label) { id = -1; this.label = label; @@ -389,6 +403,31 @@ public void testConstructorFieldProtocolOrderCodegen() { assertEquals(buffer.readInt32(), 42); } + @Test + public void testRegisterConstructor() throws Exception { + Constructor constructor = + RegisteredCtorBean.class.getDeclaredConstructor(int.class, String.class); + for (boolean codegen : new boolean[] {false, true}) { + Fory fory = + Fory.builder() + .withXlang(false) + .withRefTracking(true) + .withCodegen(codegen) + .requireClassRegistration(false) + .build(); + fory.registerConstructor(RegisteredCtorBean.class, constructor, "age", "name"); + assertEquals( + fory.getTypeResolver() + .getObjectCreator(RegisteredCtorBean.class) + .getConstructorFieldNames(), + new String[] {"age", "name"}); + RegisteredCtorBean value = new RegisteredCtorBean(42, "amy"); + RegisteredCtorBean newValue = (RegisteredCtorBean) fory.deserialize(fory.serialize(value)); + assertEquals(newValue.name, value.name); + assertEquals(newValue.age, value.age); + } + } + @Test public void testCtorInterveningRef() { Fory fory = @@ -772,14 +811,8 @@ public void testSerialization() { @Test public void testAndroidObjectSerializerReflectionPaths() throws Exception { - String javaBin = - System.getProperty("java.home") + File.separator + "bin" + File.separator + "java"; ProcessBuilder processBuilder = - new ProcessBuilder( - javaBin, - "-cp", - System.getProperty("java.class.path"), - AndroidObjectSerializerProbe.class.getName()) + new ProcessBuilder(TestUtils.javaCommand(AndroidObjectSerializerProbe.class)) .redirectErrorStream(true); processBuilder.environment().put("FORY_ANDROID_ENABLED", "1"); Process process = processBuilder.start(); diff --git a/java/fory-core/src/test/java/org/apache/fory/serializer/collection/CollectionSerializersTest.java b/java/fory-core/src/test/java/org/apache/fory/serializer/collection/CollectionSerializersTest.java index 7eb639c0d1..7f6e648f79 100644 --- a/java/fory-core/src/test/java/org/apache/fory/serializer/collection/CollectionSerializersTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/serializer/collection/CollectionSerializersTest.java @@ -28,7 +28,6 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import com.google.common.collect.ImmutableSortedSet; -import java.beans.ConstructorProperties; import java.io.Externalizable; import java.io.IOException; import java.io.ObjectInput; @@ -71,6 +70,7 @@ import lombok.EqualsAndHashCode; import org.apache.fory.Fory; import org.apache.fory.ForyTestBase; +import org.apache.fory.annotation.ForyConstructor; import org.apache.fory.context.ReadContext; import org.apache.fory.exception.DeserializationException; import org.apache.fory.memory.MemoryBuffer; @@ -886,7 +886,7 @@ public static class CollectionViewTestStruct { Collection collection; Set set; - @ConstructorProperties({"collection", "set"}) + @ForyConstructor({"collection", "set"}) public CollectionViewTestStruct(Collection collection, Set set) { this.collection = collection; this.set = set; @@ -1381,7 +1381,7 @@ public TestClassForDefaultCollectionSerializer() { this(new ArrayList<>()); } - @ConstructorProperties({"data"}) + @ForyConstructor({"data"}) public TestClassForDefaultCollectionSerializer(List data) { this.data = data; } diff --git a/java/fory-core/src/test/java/org/apache/fory/serializer/collection/MapSerializersTest.java b/java/fory-core/src/test/java/org/apache/fory/serializer/collection/MapSerializersTest.java index 35c3f9ab9c..dba05453e5 100644 --- a/java/fory-core/src/test/java/org/apache/fory/serializer/collection/MapSerializersTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/serializer/collection/MapSerializersTest.java @@ -26,7 +26,6 @@ import static org.testng.Assert.assertEquals; import com.google.common.collect.ImmutableMap; -import java.beans.ConstructorProperties; import java.io.Externalizable; import java.io.IOException; import java.io.ObjectInput; @@ -55,6 +54,7 @@ import lombok.Data; import org.apache.fory.Fory; import org.apache.fory.ForyTestBase; +import org.apache.fory.annotation.ForyConstructor; import org.apache.fory.annotation.Ref; import org.apache.fory.collection.LazyMap; import org.apache.fory.collection.MapEntry; @@ -859,7 +859,7 @@ public TestClass1ForDefaultMap() { this(new HashSet<>()); } - @ConstructorProperties({"data"}) + @ForyConstructor({"data"}) public TestClass1ForDefaultMap(Set data) { this.data = data; } @@ -883,7 +883,7 @@ public TestClass2ForDefaultMap() { this(new HashSet<>()); } - @ConstructorProperties({"data"}) + @ForyConstructor({"data"}) public TestClass2ForDefaultMap(Set> data) { this.data = data; } @@ -1227,7 +1227,7 @@ public static class LazyMapCollectionFieldStruct { List> mapList; PrivateMap map; - @ConstructorProperties({"mapList", "map"}) + @ForyConstructor({"mapList", "map"}) LazyMapCollectionFieldStruct( List> mapList, PrivateMap map) { this.mapList = mapList; @@ -1446,7 +1446,7 @@ public PrivateFinalMapFieldStruct() { this(new LinkedHashMap<>(), new LinkedHashMap<>()); } - @ConstructorProperties({"valueMap", "keyMap"}) + @ForyConstructor({"valueMap", "keyMap"}) public PrivateFinalMapFieldStruct( Map valueMap, Map keyMap) { this.valueMap = valueMap; diff --git a/java/pom.xml b/java/pom.xml index 0e49f92add..358e8493e2 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -72,6 +72,8 @@ UTF-8 32.1.2-jre 3.1.12 + 3.0.2 + 2.0.12 1.13 ${basedir} @@ -110,38 +112,6 @@ fory-latest-jdk-tests - - - jdk25-test-classpath - - [25,] - - fory.jdk25.test.classpath - true - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - - org.apache.fory:fory-core - - - - ${maven.multiModuleProjectDirectory}/fory-core/target/jdk25-test-classes - - - - - - - @@ -149,7 +119,7 @@ org.slf4j slf4j-api - 2.0.12 + ${slf4j.version} provided true diff --git a/kotlin/fory-kotlin-ksp/src/main/kotlin/org/apache/fory/kotlin/ksp/ForyKotlinSymbolProcessor.kt b/kotlin/fory-kotlin-ksp/src/main/kotlin/org/apache/fory/kotlin/ksp/ForyKotlinSymbolProcessor.kt index 245dbb9d27..8855925932 100644 --- a/kotlin/fory-kotlin-ksp/src/main/kotlin/org/apache/fory/kotlin/ksp/ForyKotlinSymbolProcessor.kt +++ b/kotlin/fory-kotlin-ksp/src/main/kotlin/org/apache/fory/kotlin/ksp/ForyKotlinSymbolProcessor.kt @@ -48,6 +48,14 @@ private data class ParsedStructFields( val fields: List, ) +private sealed class ConstructorFields { + object Absent : ConstructorFields() + + object Invalid : ConstructorFields() + + data class Present(val names: List) : ConstructorFields() +} + internal fun fieldLimitError(defaultFieldCount: Int): String? = if (defaultFieldCount > MAX_DEFAULT_FIELDS) { "Kotlin KSP xlang serializers currently support at most $MAX_DEFAULT_FIELDS defaulted constructor fields because Kotlin source generation must call constructors with omitted default arguments" @@ -100,6 +108,24 @@ internal fun ctorVisibilityError(modifiers: Set): String? = null } +internal fun constructorBindingError( + parameterNames: List, + fieldNames: List, + targetName: String, +): String? { + if (parameterNames.size != fieldNames.size) { + return "@ForyConstructor on $targetName must declare one field name for each primary constructor parameter" + } + val duplicates = fieldNames.groupingBy { it }.eachCount().filterValues { it > 1 }.keys + if (duplicates.isNotEmpty()) { + return "@ForyConstructor on $targetName declares duplicate field name ${duplicates.first()}" + } + if (fieldNames.any { it.isBlank() }) { + return "@ForyConstructor on $targetName must not declare blank field names" + } + return null +} + internal class ForyKotlinSymbolProcessor(private val environment: SymbolProcessorEnvironment) : SymbolProcessor { private val codeGenerator: CodeGenerator = environment.codeGenerator @@ -258,46 +284,137 @@ internal class ForyKotlinSymbolProcessor(private val environment: SymbolProcesso declaration: KSClassDeclaration, primaryConstructor: KSFunctionDeclaration, ): ParsedStructFields? { + val constructorFields = + when (val parsed = parseConstructorFields(declaration, primaryConstructor)) { + ConstructorFields.Absent -> null + ConstructorFields.Invalid -> return null + is ConstructorFields.Present -> parsed.names + } if (primaryConstructor.parameters.isEmpty()) { + if (constructorFields != null) { + logger.error( + "@ForyConstructor is only supported on Kotlin primary constructors that build schema fields", + primaryConstructor, + ) + return null + } return ParsedStructFields( KotlinStructConstruction.MUTABLE, parseVarFields(declaration) ?: return null, ) } val propertiesByName = declaration.getAllProperties().associateBy { it.simpleName.asString() } - val fields = parseCtorFields(declaration, primaryConstructor, propertiesByName) ?: return null + val fields = + parseCtorFields(declaration, primaryConstructor, propertiesByName, constructorFields) + ?: return null return ParsedStructFields(KotlinStructConstruction.CONSTRUCTOR, fields) } + private fun parseConstructorFields( + declaration: KSClassDeclaration, + primaryConstructor: KSFunctionDeclaration, + ): ConstructorFields { + val annotation = + primaryConstructor.annotations.firstOrNull { isAnnotation(it, FORY_CONSTRUCTOR) } + ?: return ConstructorFields.Absent + val fieldNames = mutableListOf() + val argument = + annotation.arguments.firstOrNull { it.name?.asString() == "value" } + ?: annotation.arguments.singleOrNull() + when (val value = argument?.value) { + is String -> fieldNames.add(value) + is List<*> -> { + for (entry in value) { + if (entry !is String) { + logger.error("@ForyConstructor values must be field names", primaryConstructor) + return ConstructorFields.Invalid + } + fieldNames.add(entry) + } + } + is Array<*> -> { + for (entry in value) { + if (entry !is String) { + logger.error("@ForyConstructor values must be field names", primaryConstructor) + return ConstructorFields.Invalid + } + fieldNames.add(entry) + } + } + else -> { + logger.error("@ForyConstructor must declare field names", primaryConstructor) + return ConstructorFields.Invalid + } + } + val parameterNames = mutableListOf() + for (parameter in primaryConstructor.parameters) { + val parameterName = parameter.name?.asString() + if (parameterName == null) { + logger.error( + "@ForyConstructor requires named Kotlin primary constructor parameters", + parameter, + ) + return ConstructorFields.Invalid + } + parameterNames.add(parameterName) + } + val bindingError = + constructorBindingError( + parameterNames, + fieldNames, + declaration.qualifiedName?.asString() ?: declaration.simpleName.asString(), + ) + if (bindingError != null) { + logger.error(bindingError, primaryConstructor) + return ConstructorFields.Invalid + } + return ConstructorFields.Present(fieldNames) + } + private fun parseCtorFields( declaration: KSClassDeclaration, primaryConstructor: KSFunctionDeclaration, propertiesByName: Map, + constructorFields: List?, ): List? { val fields = mutableListOf() val foryIds = hashSetOf() var nextId = 0 - for (parameter in primaryConstructor.parameters) { + val explicitConstructor = constructorFields != null + for ((index, parameter) in primaryConstructor.parameters.withIndex()) { val parameterName = parameter.name?.asString() ?: continue - val property = propertiesByName[parameterName] - if (property == null || (!parameter.isVal && !parameter.isVar)) { + val fieldName = constructorFields?.get(index) ?: parameterName + val property = propertiesByName[fieldName] + if (property == null || (!explicitConstructor && !parameter.isVal && !parameter.isVar)) { logger.error( - "Constructor parameter $parameterName is not a field-backed property", + "Constructor parameter $parameterName is not bound to an accessible schema property", parameter ) return null } + val fieldType = property.type.resolve() + val parameterType = parameter.type.resolve() + val fieldTypeName = kotlinSourceTypeName(fieldType) + val parameterTypeName = kotlinSourceTypeName(parameterType) + if (explicitConstructor && fieldTypeName != parameterTypeName) { + logger.error( + "@ForyConstructor field $fieldName type $fieldTypeName must match primary constructor parameter $parameterName type $parameterTypeName", + parameter, + ) + return null + } val field = parseField( declaration, property, - parameterName, - parameter.type.resolve(), + fieldName, + fieldType, parameter, nextId, parameter.hasDefault, foryIds, requireForyId = false, + constructorParameterName = parameterName, ) ?: return null fields.add(field) nextId++ @@ -336,6 +453,7 @@ internal class ForyKotlinSymbolProcessor(private val environment: SymbolProcesso hasDefault = false, foryIds, requireForyId = true, + constructorParameterName = property.simpleName.asString(), ) ?: return null fields.add(field) } @@ -352,6 +470,7 @@ internal class ForyKotlinSymbolProcessor(private val environment: SymbolProcesso hasDefault: Boolean, foryIds: MutableSet, requireForyId: Boolean, + constructorParameterName: String, ): KotlinSourceField? { if (Modifier.PRIVATE in property.modifiers) { logger.error("Private Fory field $fieldName is inaccessible to generated code", property) @@ -382,6 +501,7 @@ internal class ForyKotlinSymbolProcessor(private val environment: SymbolProcesso return KotlinSourceField( id = id, name = fieldName, + constructorParameterName = constructorParameterName, type = typeNode, hasForyField = fieldMeta.hasAnnotation, foryFieldId = fieldMeta.id, @@ -448,10 +568,7 @@ internal class ForyKotlinSymbolProcessor(private val environment: SymbolProcesso } val caseId = foryCaseId(caseDeclaration, reportMissing = false) ?: continue if (caseId < 0) { - logger.error( - "Schema Kotlin union @ForyCase ids must be non-negative", - caseDeclaration - ) + logger.error("Schema Kotlin union @ForyCase ids must be non-negative", caseDeclaration) return null } if (!caseIds.add(caseId)) { @@ -1710,6 +1827,7 @@ internal class ForyKotlinSymbolProcessor(private val environment: SymbolProcesso private companion object { const val FORY_STRUCT = "org.apache.fory.annotation.ForyStruct" + const val FORY_CONSTRUCTOR = "org.apache.fory.annotation.ForyConstructor" const val FORY_UNION = "org.apache.fory.annotation.ForyUnion" const val FORY_CASE = "org.apache.fory.annotation.ForyCase" const val FORY_UNKNOWN_CASE = "org.apache.fory.annotation.ForyUnknownCase" diff --git a/kotlin/fory-kotlin-ksp/src/main/kotlin/org/apache/fory/kotlin/ksp/KotlinSerializerSourceWriter.kt b/kotlin/fory-kotlin-ksp/src/main/kotlin/org/apache/fory/kotlin/ksp/KotlinSerializerSourceWriter.kt index 00ba8870a8..6c91ae89c0 100644 --- a/kotlin/fory-kotlin-ksp/src/main/kotlin/org/apache/fory/kotlin/ksp/KotlinSerializerSourceWriter.kt +++ b/kotlin/fory-kotlin-ksp/src/main/kotlin/org/apache/fory/kotlin/ksp/KotlinSerializerSourceWriter.kt @@ -92,6 +92,8 @@ internal class KotlinSerializerSourceWriter(private val struct: KotlinSourceStru builder.append(" private val allFields: Array\n") builder.append(" private val allFieldIds: IntArray\n") builder.append(" private val fieldsById: Array\n") + builder.append(" private val constructorFieldIds: IntArray?\n") + builder.append(" private val constructorFieldBits: LongArray?\n") builder.append(" private val classVersionHash: Int\n") builder.append(" private val sameSchemaCompatible: Boolean\n\n") } @@ -134,6 +136,8 @@ internal class KotlinSerializerSourceWriter(private val struct: KotlinSourceStru builder.append(" this.allFields = emptyArray()\n") builder.append(" this.allFieldIds = IntArray(0)\n") builder.append(" this.fieldsById = arrayOfNulls(0)\n") + builder.append(" this.constructorFieldIds = null\n") + builder.append(" this.constructorFieldBits = null\n") builder.append(" this.classVersionHash = 0\n") builder.append(" this.sameSchemaCompatible = false\n") builder.append(" }\n\n") @@ -162,6 +166,12 @@ internal class KotlinSerializerSourceWriter(private val struct: KotlinSourceStru builder.append(" for (i in this.allFields.indices) {\n") builder.append(" this.fieldsById[this.allFieldIds[i]] = this.allFields[i]\n") builder.append(" }\n") + builder.append( + " this.constructorFieldIds = if (objectCreator.hasConstructorFields()) buildConstructorFieldIds(DESCRIPTORS) else null\n" + ) + builder.append( + " this.constructorFieldBits = buildConstructorFieldBits(DESCRIPTORS.size, constructorFieldIds)\n" + ) writeScalarBindings() builder.append( " this.classVersionHash = if (typeResolver.checkClassVersion()) computeClassVersionHash(DESCRIPTORS) else 0\n" @@ -234,6 +244,177 @@ internal class KotlinSerializerSourceWriter(private val struct: KotlinSourceStru builder.append(" }\n") builder.append(" }\n") builder.append(" }\n\n") + + builder + .append(" private fun readCompatibleConstructor(readContext: ReadContext): ") + .append(struct.typeName) + .append(" {\n") + builder.append(" val fieldValues = arrayOfNulls(DESCRIPTORS.size)\n") + builder.append(" val bufferedFields = newFieldBits(DESCRIPTORS.size)\n") + builder.append(" val presentFields = newFieldBits(DESCRIPTORS.size)\n") + builder.append(" beginConstructorRef(readContext)\n") + builder.append(" try {\n") + builder.append(" var remaining = countConstructorFields(constructorFieldBits!!)\n") + builder.append(" var value: ").append(struct.typeName).append("? = null\n") + builder.append(" if (remaining == 0) {\n") + builder.append(" val constructed = newConstructorObject(fieldValues)\n") + builder.append(" value = constructed\n") + builder.append(" referenceConstructorRef(readContext, constructed)\n") + builder.append(" }\n") + builder.append(" for (i in remoteFields.indices) {\n") + builder.append(" val remoteField = remoteFields[i]\n") + builder.append(" val fieldId = remoteField.matchedId\n") + builder.append(" if (fieldId < 0) {\n") + builder.append(" skipField(readContext, remoteField)\n") + builder.append(" continue\n") + builder.append(" }\n") + builder.append(" val localField = fieldsById[fieldId]!!\n") + builder.append(" if (!canReadRemoteField(remoteField, localField)) {\n") + builder.append(" skipField(readContext, remoteField)\n") + builder.append(" continue\n") + builder.append(" }\n") + builder.append( + " val fieldValue = readCompatibleConstructorField(readContext, remoteField, localField, fieldId)\n" + ) + builder.append(" markField(presentFields, fieldId)\n") + builder.append(" if (hasField(constructorFieldBits!!, fieldId)) {\n") + builder.append( + " fieldValues[fieldId] = ctorFieldValue(readContext, fieldValue, type)\n" + ) + builder.append(" remaining--\n") + builder.append(" if (remaining == 0) {\n") + builder.append(" checkNoUnresolvedReadRef(readContext)\n") + builder.append(" val constructed = newConstructorObject(fieldValues)\n") + builder.append(" value = constructed\n") + builder.append(" referenceConstructorRef(readContext, constructed)\n") + builder.append(" setBufferedFields(constructed, fieldValues, bufferedFields)\n") + builder.append(" }\n") + builder.append(" } else if (value == null) {\n") + builder.append( + " fieldValues[fieldId] = bufferFieldValue(readContext, fieldValue, type)\n" + ) + builder.append(" markField(bufferedFields, fieldId)\n") + builder.append(" } else {\n") + builder.append(" setFieldById(value!!, localField, fieldId, fieldValue)\n") + builder.append(" }\n") + builder.append(" }\n") + for (field in struct.fields) { + if (field.hasDefault || field.nullable) { + continue + } + builder.append(" if (!hasField(presentFields, ").append(field.id).append(")) {\n") + builder + .append(" throw DeserializationException(\"Required Kotlin field ") + .append(struct.qualifiedTypeName) + .append('.') + .append(field.name) + .append(" is missing in compatible xlang payload\")\n") + builder.append(" }\n") + } + builder.append(" if (value == null) {\n") + builder.append(" checkNoUnresolvedReadRef(readContext)\n") + builder.append(" val constructed = newConstructorObject(fieldValues)\n") + builder.append(" value = constructed\n") + builder.append(" referenceConstructorRef(readContext, constructed)\n") + builder.append(" setBufferedFields(constructed, fieldValues, bufferedFields)\n") + builder.append(" }\n") + builder.append(" return value!!\n") + builder.append(" } finally {\n") + builder.append(" endConstructorRef(readContext)\n") + builder.append(" }\n") + builder.append(" }\n\n") + + builder.append( + " private fun readCompatibleConstructorField(readContext: ReadContext, remoteField: StaticGeneratedStructSerializer.RemoteFieldInfo, localField: SerializationFieldInfo, fieldId: Int): Any? {\n" + ) + builder.append(" return when (fieldId) {\n") + for (field in struct.fields) { + val expression = + castReadExpression( + field, + "readCompatibleFieldValue(readContext, remoteField, localField)", + compatible = true, + ) + builder.append(" ").append(field.id).append(" -> ").append(expression).append("\n") + } + builder.append( + " else -> throw IllegalStateException(\"Unknown generated field id \${fieldId}\")\n" + ) + builder.append(" }\n") + builder.append(" }\n\n") + + builder + .append(" private fun newConstructorObject(fieldValues: Array): ") + .append(struct.typeName) + .append(" {\n") + builder.append( + " return objectCreator.newInstanceWithArguments(*constructorArgs(fieldValues, constructorFieldIds!!, objectCreator.getConstructorFieldTypes())) as " + ) + builder.append(struct.typeName).append("\n") + builder.append(" }\n\n") + + builder.append( + " private fun setBufferedFields(value: ${struct.typeName}, fieldValues: Array, bufferedFields: LongArray) {\n" + ) + builder.append(" for (fieldId in fieldsById.indices) {\n") + builder.append(" if (hasField(bufferedFields, fieldId)) {\n") + builder.append( + " setFieldById(value, fieldsById[fieldId]!!, fieldId, resolveBufferedValue(fieldValues[fieldId], value))\n" + ) + builder.append(" }\n") + builder.append(" }\n") + builder.append(" }\n\n") + + builder.append( + " private fun setFieldById(value: ${struct.typeName}, fieldInfo: SerializationFieldInfo, fieldId: Int, fieldValue: Any?) {\n" + ) + builder.append(" when (fieldId) {\n") + for (field in struct.fields) { + builder + .append(" ") + .append(field.id) + .append(" -> setGeneratedFieldValue(value, fieldInfo, fieldValue)\n") + } + builder.append( + " else -> throw IllegalStateException(\"Unknown generated field id \${fieldId}\")\n" + ) + builder.append(" }\n") + builder.append(" }\n\n") + + builder + .append(" private fun copyConstructorObject(copyContext: CopyContext, value: ") + .append(struct.typeName) + .append("): ") + .append(struct.typeName) + .append(" {\n") + builder.append(" val fieldValues = arrayOfNulls(DESCRIPTORS.size)\n") + for (field in struct.fields) { + builder.append(" if (hasField(constructorFieldBits!!, ").append(field.id).append(")) {\n") + builder + .append(" fieldValues[") + .append(field.id) + .append("] = copyConstructorFieldValue(copyContext, value, ") + .append("value.") + .append(field.name) + .append(", fieldsById[") + .append(field.id) + .append("]!!)\n") + builder.append(" }\n") + } + builder.append(" val copied = newConstructorObject(fieldValues)\n") + builder.append(" copyContext.reference(value, copied)\n") + for (field in struct.fields) { + builder.append(" if (!hasField(constructorFieldBits!!, ").append(field.id).append(")) {\n") + builder + .append(" setGeneratedFieldValue(copied, fieldsById[") + .append(field.id) + .append("]!!, ") + .append(copyExpression(field)) + .append(")\n") + builder.append(" }\n") + } + builder.append(" return copied\n") + builder.append(" }\n\n") } private fun writeRead() { @@ -257,9 +438,13 @@ internal class KotlinSerializerSourceWriter(private val struct: KotlinSourceStru builder.append(" if (typeResolver.checkClassVersion()) {\n") builder.append(" checkClassVersion(buffer.readInt32(), classVersionHash)\n") builder.append(" }\n") + builder.append(" if (constructorFieldIds != null) {\n") + builder.append(" return readSchemaConstructor(readContext)\n") + builder.append(" }\n") if (struct.construction == KotlinStructConstruction.MUTABLE) { writeMutableReadBody() builder.append(" }\n\n") + writeConstructorRead() return } writeLocalDeclarations() @@ -288,6 +473,80 @@ internal class KotlinSerializerSourceWriter(private val struct: KotlinSourceStru appendConstructorCall(defaultMask = 0L) builder.append("\n") builder.append(" }\n\n") + writeConstructorRead() + } + + private fun writeConstructorRead() { + builder + .append(" private fun readSchemaConstructor(readContext: ReadContext): ") + .append(struct.typeName) + .append(" {\n") + builder.append(" val fieldValues = arrayOfNulls(DESCRIPTORS.size)\n") + builder.append(" val bufferedFields = newFieldBits(DESCRIPTORS.size)\n") + builder.append(" beginConstructorRef(readContext)\n") + builder.append(" try {\n") + builder.append(" var remaining = countConstructorFields(constructorFieldBits!!)\n") + builder.append(" var value: ").append(struct.typeName).append("? = null\n") + builder.append(" if (remaining == 0) {\n") + builder.append(" val constructed = newConstructorObject(fieldValues)\n") + builder.append(" value = constructed\n") + builder.append(" referenceConstructorRef(readContext, constructed)\n") + builder.append(" }\n") + builder.append(" for (i in allFields.indices) {\n") + builder.append(" val fieldInfo = allFields[i]\n") + builder.append(" val fieldId = allFieldIds[i]\n") + builder.append( + " val fieldValue = readSchemaConstructorField(readContext, fieldInfo, fieldId)\n" + ) + builder.append(" if (hasField(constructorFieldBits!!, fieldId)) {\n") + builder.append( + " fieldValues[fieldId] = ctorFieldValue(readContext, fieldValue, type)\n" + ) + builder.append(" remaining--\n") + builder.append(" if (remaining == 0) {\n") + builder.append(" checkNoUnresolvedReadRef(readContext)\n") + builder.append(" val constructed = newConstructorObject(fieldValues)\n") + builder.append(" value = constructed\n") + builder.append(" referenceConstructorRef(readContext, constructed)\n") + builder.append(" setBufferedFields(constructed, fieldValues, bufferedFields)\n") + builder.append(" }\n") + builder.append(" } else if (value == null) {\n") + builder.append( + " fieldValues[fieldId] = bufferFieldValue(readContext, fieldValue, type)\n" + ) + builder.append(" markField(bufferedFields, fieldId)\n") + builder.append(" } else {\n") + builder.append(" setFieldById(value!!, fieldInfo, fieldId, fieldValue)\n") + builder.append(" }\n") + builder.append(" }\n") + builder.append(" if (value == null) {\n") + builder.append(" checkNoUnresolvedReadRef(readContext)\n") + builder.append(" val constructed = newConstructorObject(fieldValues)\n") + builder.append(" value = constructed\n") + builder.append(" referenceConstructorRef(readContext, constructed)\n") + builder.append(" setBufferedFields(constructed, fieldValues, bufferedFields)\n") + builder.append(" }\n") + builder.append(" return value!!\n") + builder.append(" } finally {\n") + builder.append(" endConstructorRef(readContext)\n") + builder.append(" }\n") + builder.append(" }\n\n") + + builder.append( + " private fun readSchemaConstructorField(readContext: ReadContext, fieldInfo: SerializationFieldInfo, fieldId: Int): Any? {\n" + ) + builder.append(" val buffer = readContext.buffer\n") + builder.append(" return when (fieldId) {\n") + for (field in struct.fields) { + val direct = directReadExpression(field) + val expression = direct ?: castReadExpression(field, "readFieldValue(readContext, fieldInfo)") + builder.append(" ").append(field.id).append(" -> ").append(expression).append("\n") + } + builder.append( + " else -> throw IllegalStateException(\"Unknown generated field id \${fieldId}\")\n" + ) + builder.append(" }\n") + builder.append(" }\n\n") } private fun writeMutableReadBody() { @@ -326,6 +585,9 @@ internal class KotlinSerializerSourceWriter(private val struct: KotlinSourceStru builder.append(" if (sameSchemaCompatible) {\n") builder.append(" return readSchemaConsistent(readContext)\n") builder.append(" }\n") + builder.append(" if (constructorFieldIds != null) {\n") + builder.append(" return readCompatibleConstructor(readContext)\n") + builder.append(" }\n") if (struct.construction == KotlinStructConstruction.MUTABLE) { writeMutableCompatibleReadBody() builder.append(" }\n\n") @@ -488,6 +750,9 @@ internal class KotlinSerializerSourceWriter(private val struct: KotlinSourceStru builder.append(" if (immutable) {\n") builder.append(" return value\n") builder.append(" }\n") + builder.append(" if (constructorFieldIds != null) {\n") + builder.append(" return copyConstructorObject(copyContext, value)\n") + builder.append(" }\n") if (struct.construction == KotlinStructConstruction.MUTABLE) { writeMutableCopyBody() builder.append(" }\n") @@ -569,6 +834,20 @@ internal class KotlinSerializerSourceWriter(private val struct: KotlinSourceStru builder.append(" return copy\n") } + private fun copyExpression(field: KotlinSourceField): String = + when { + field.type.primitive || isScalarUnsigned(field) -> "value.${field.name}" + isDenseUnsignedArray(field) -> + if (field.nullable) "value.${field.name}?.copyOf()" else "value.${field.name}.copyOf()" + field.type.isCollectionOrMap() -> + copyContainerExpression(field.type, "value.${field.name}", 0) + else -> + castReadExpression( + field, + "copyFieldValue(copyContext, value.${field.name}, fieldsById[${field.id}]!!)" + ) + } + private fun writeLocalDeclarations() { for (field in struct.fields) { builder @@ -624,7 +903,10 @@ internal class KotlinSerializerSourceWriter(private val struct: KotlinSourceStru if (!first) { builder.append(", ") } - builder.append(field.name).append(" = ").append(constructorValueExpression(field)) + builder + .append(field.constructorParameterName) + .append(" = ") + .append(constructorValueExpression(field)) first = false } builder.append(")") diff --git a/kotlin/fory-kotlin-ksp/src/main/kotlin/org/apache/fory/kotlin/ksp/Model.kt b/kotlin/fory-kotlin-ksp/src/main/kotlin/org/apache/fory/kotlin/ksp/Model.kt index 687f150bfb..a50b0ca62f 100644 --- a/kotlin/fory-kotlin-ksp/src/main/kotlin/org/apache/fory/kotlin/ksp/Model.kt +++ b/kotlin/fory-kotlin-ksp/src/main/kotlin/org/apache/fory/kotlin/ksp/Model.kt @@ -81,6 +81,7 @@ internal data class KotlinSourceField( val hasDefault: Boolean, val nullable: Boolean, val propertyTypeName: String, + val constructorParameterName: String = name, ) { val localName: String = "field$id" } diff --git a/kotlin/fory-kotlin-ksp/src/test/kotlin/org/apache/fory/kotlin/ksp/ProcessorValidationTest.kt b/kotlin/fory-kotlin-ksp/src/test/kotlin/org/apache/fory/kotlin/ksp/ProcessorValidationTest.kt index 21f7e4fc58..477b6eb04c 100644 --- a/kotlin/fory-kotlin-ksp/src/test/kotlin/org/apache/fory/kotlin/ksp/ProcessorValidationTest.kt +++ b/kotlin/fory-kotlin-ksp/src/test/kotlin/org/apache/fory/kotlin/ksp/ProcessorValidationTest.kt @@ -75,6 +75,74 @@ class ProcessorValidationTest { ) } + @Test + fun validatesConstructorBinding() { + assertNull(constructorBindingError(listOf("userName"), listOf("name"), "example.User")) + assertEquals( + constructorBindingError(listOf("name"), listOf("name", "age"), "example.User"), + "@ForyConstructor on example.User must declare one field name for each primary constructor parameter", + ) + assertEquals( + constructorBindingError(listOf("name", "age"), listOf("name", "name"), "example.User"), + "@ForyConstructor on example.User declares duplicate field name name", + ) + assertEquals( + constructorBindingError(listOf("name"), listOf(""), "example.User"), + "@ForyConstructor on example.User must not declare blank field names", + ) + } + + @Test + fun constructorBindingNamesArguments() { + val stringType = + KotlinSourceTypeNode( + rawClassExpression = "String::class.java", + kotlinTypeName = "kotlin.String", + valueTypeName = "String", + typeName = "java.lang.String", + typeId = "Types.STRING", + nullable = false, + trackingRef = false, + primitive = false, + unsigned = false, + ) + val source = + KotlinSerializerSourceWriter( + KotlinSourceStruct( + packageName = "example", + typeName = "User", + qualifiedTypeName = "example.User", + serializerName = "User_ForySerializer", + serializerVisibility = KotlinSerializerVisibility.PUBLIC, + fields = + listOf( + KotlinSourceField( + id = 0, + name = "name", + type = stringType, + hasForyField = true, + foryFieldId = 1, + trackingRef = false, + dynamic = "AUTO", + arrayType = false, + hasDefault = false, + nullable = false, + propertyTypeName = "String", + constructorParameterName = "userName", + ) + ), + originatingFiles = emptyList(), + ) + ) + .write() + + assertTrue(source.contains("writeContext.writeString(value.name)")) + assertTrue(source.contains("return User(userName = field0!!)")) + assertTrue(!source.contains("return User(name = field0!!)")) + assertTrue(source.contains("constructorFieldIds = if (objectCreator.hasConstructorFields())")) + assertTrue(source.contains("objectCreator.newInstanceWithArguments(*constructorArgs")) + } + @Test fun tracksWidePresence() { val intType = diff --git a/kotlin/fory-kotlin-tests/src/main/kotlin/org/apache/fory/kotlin/xlang/KotlinXlangPeer.kt b/kotlin/fory-kotlin-tests/src/main/kotlin/org/apache/fory/kotlin/xlang/KotlinXlangPeer.kt index 0608fbc813..563c311f62 100644 --- a/kotlin/fory-kotlin-tests/src/main/kotlin/org/apache/fory/kotlin/xlang/KotlinXlangPeer.kt +++ b/kotlin/fory-kotlin-tests/src/main/kotlin/org/apache/fory/kotlin/xlang/KotlinXlangPeer.kt @@ -56,6 +56,12 @@ public data class KotlinUser( @ForyField(id = 3) val score: @VarInt Long, ) +@ForyStruct +public data class KotlinRegisteredSwap( + @ForyField(id = 1) val left: String, + @ForyField(id = 2) val right: String, +) + @ForyStruct internal data class KotlinInternalUser( @ForyField(id = 1) val id: UInt, @@ -167,7 +173,14 @@ private fun staticSerializerRoundTrip(dataFile: String) { checkNoArgRegisterReceivers() val fory = newFory() + fory.registerConstructor( + KotlinRegisteredSwap::class.java, + KotlinRegisteredSwap::class.java.getDeclaredConstructor(String::class.java, String::class.java), + "right", + "left", + ) fory.register("kotlin", "KotlinUser") + fory.register("kotlin", "KotlinRegisteredSwap") fory.register("kotlin", "KotlinInternalUser") fory.register("kotlin", "KotlinConcreteCollections") fory.register("kotlin", "KotlinUnsignedCollections") @@ -196,6 +209,14 @@ private fun staticSerializerRoundTrip(dataFile: String) { check(descriptors[0].typeRef.typeExtMeta.typeId() == Types.UINT32) check(descriptors[2].typeRef.typeExtMeta.typeId() == Types.VARINT64) + val swap = KotlinRegisteredSwap(left = "left", right = "right") + val swapped = KotlinRegisteredSwap(left = "right", right = "left") + check(fory.deserialize(fory.serialize(swap), KotlinRegisteredSwap::class.java) == swapped) + check(fory.copy(swap) == swapped) + check(fory.getSerializer(KotlinRegisteredSwap::class.java) is StaticGeneratedStructSerializer<*>) { + "KotlinRegisteredSwap did not load a static generated serializer" + } + val internalUser = KotlinInternalUser(id = UInt.MAX_VALUE, name = "internal-static") check( fory.deserialize(fory.serialize(internalUser), KotlinInternalUser::class.java) == internalUser diff --git a/scala/build.sbt b/scala/build.sbt index 63407932f2..1015c20fa4 100644 --- a/scala/build.sbt +++ b/scala/build.sbt @@ -49,12 +49,7 @@ libraryDependencies ++= Seq( Test / fork := true Test / javaOptions ++= Seq( - "--add-opens=java.base/java.lang=ALL-UNNAMED", "--add-opens=java.base/java.lang.invoke=ALL-UNNAMED", - "--add-opens=java.base/java.lang.reflect=ALL-UNNAMED", - "--add-opens=java.base/jdk.internal.reflect=ALL-UNNAMED", - "--add-opens=java.base/java.util=ALL-UNNAMED", - "--add-opens=java.base/java.util.concurrent=ALL-UNNAMED", ) lazy val writeTestClasspath = taskKey[File]("Writes the Scala test runtime classpath") From 050afdb062bf0e6d8c2b34665ee0a43ebe65b06e Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Tue, 2 Jun 2026 00:54:52 +0800 Subject: [PATCH 37/69] feat(java): remove unsafe requirement on jdk25 --- .agents/languages/java.md | 4 +- .github/workflows/ci.yml | 2 +- benchmarks/java/pom.xml | 10 +-- .../fory/benchmark/Jdk25MrJarCheck.java | 24 ++++-- .../apache/fory/benchmark/MemorySuite.java | 10 --- .../fory/benchmark/NewJava11StringSuite.java | 32 +------ .../apache/fory/benchmark/NewStringSuite.java | 21 ----- .../apache/fory/benchmark/UnsafeAccess.java | 37 -------- benchmarks/java25/README.md | 18 +--- .../java25/DirectMemoryAccessBenchmark.java | 62 +------------- .../java25/DirectToHeapCopyBenchmark.java | 43 +--------- ci/run_ci.sh | 21 ++--- ci/tasks/java.py | 45 ++++++---- docs/guide/java/native-serialization.md | 4 +- docs/guide/java/troubleshooting.md | 15 +--- docs/guide/kotlin/configuration.md | 17 +++- integration_tests/graalvm_tests/pom.xml | 35 +------- .../fory/graalvm/FeatureTestExample.java | 4 +- .../java/org/apache/fory/graalvm/Foo.java | 4 +- .../fory/graalvm/ObjectStreamExample.java | 4 +- .../jdk_compatibility_tests/pom.xml | 23 ++--- .../StaticSerializerSourceWriter.java | 6 +- .../serializer/AbstractObjectSerializer.java | 44 +++++++++- .../fory/serializer/CompatibleSerializer.java | 84 ++++--------------- .../fory/serializer/ObjectSerializer.java | 43 +--------- .../serializer/ObjectStreamSerializer.java | 4 +- .../StaticGeneratedStructSerializer.java | 6 +- .../java/org/apache/fory/type/TypeUtils.java | 4 +- .../src/main/java16/module-info.java | 5 -- .../src/main/java25/module-info.java | 6 -- .../fory/reflect/FieldAccessorStrategy.java | 3 +- .../fory/reflect/UnsafeObjectAllocator.java | 34 +++++--- .../fory-core/src/main/java9/module-info.java | 5 -- .../fory-core/native-image.properties | 1 + .../test/java/org/apache/fory/TestUtils.java | 2 +- .../serializer/AndroidJvmRoundTripTest.java | 1 - .../serializer/ProtobufSerializer.java | 3 +- java/fory-format/pom.xml | 1 - .../ImmutableCollectionSerializersTest.java | 4 +- .../apache/fory/test/bean/AccessBeans.java | 13 +-- .../GuavaCollectionSerializersTest.java | 4 +- .../org/apache/fory/test/FastJsonTest.java | 4 +- .../ksp/KotlinSerializerSourceWriter.kt | 7 +- scala/build.sbt | 3 - .../serializer/scala/RangeSerializer.scala | 7 +- 45 files changed, 233 insertions(+), 496 deletions(-) delete mode 100644 benchmarks/java/src/main/java/org/apache/fory/benchmark/UnsafeAccess.java diff --git a/.agents/languages/java.md b/.agents/languages/java.md index e542544ba8..6be6bb2374 100644 --- a/.agents/languages/java.md +++ b/.agents/languages/java.md @@ -56,7 +56,9 @@ Load this file when changing anything under `java/` or when Java drives a cross- - Keep JDK25+ unsafe-removal implementation invariants in agent/design docs and tests, not user guides. User guides should document user actions such as `--sun-misc-unsafe-memory-access=deny`, `java.base/java.lang.invoke` opens, JDK26+ `--enable-final-field-mutation`, package opens for user named modules, and public APIs such as `@ForyConstructor` and `BaseFory.registerConstructor(...)`; do not expose internal serializer names, owner-model rationale, or avoided fallback strategies there. - For JDK25+ zero-Unsafe final-field behavior, distinguish JDK25 from JDK26+: JDK25 has no final-field mutation flag requirement, while JDK26+ requires `--enable-final-field-mutation` for post-construction final-field writes. - For JDK25+ object creation, do not use `sun.reflect.ReflectionFactory` or `jdk.internal.reflect.ReflectionFactory`. Serializable classes without a no-arg constructor may use `ObjectStreamClass.newInstance` through the trusted lookup owner; non-Serializable classes without a no-arg constructor require `@ForyConstructor`, `BaseFory.registerConstructor(...)`, record construction, or a custom serializer. -- For JDK25+ CI, do not run core runtime tests from raw Maven reactor `target/classes`. Those classes bypass `META-INF/versions/25` and exercise the JDK8-24 root implementation. Build/install the multi-release artifact first, then test the packaged artifact through the classpath integration suite and the JPMS module-path suite. +- Constructor-bound serializers must cache constructor field metadata during serializer initialization. Do not call defensive-copy metadata getters such as `getConstructorFieldTypes()` from per-object read paths. +- For JDK25+ CI, do not run core runtime tests from raw Maven reactor `target/classes`. Those classes bypass `META-INF/versions/25` and exercise the JDK8-24 root implementation. Build/install the multi-release artifact first, then verify the zero-Unsafe path through the JPMS module-path suite where `org.apache.fory.core` is the real access target. +- Do not make GraalVM native-image JDK25+ pass by opening `java.lang.invoke` to `ALL-UNNAMED`. Keep zero-Unsafe verification on JPMS JVM tests unless the native-image path itself runs Fory as a named module and the produced binary passes. ## Key Modules diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d9c794f51c..d7baf5ab5b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -516,7 +516,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - java-version: ["17", "21", "25"] + java-version: ["17", "21"] steps: - uses: actions/checkout@v5 - uses: graalvm/setup-graalvm@f744c72a42b1995d7b0cbc314bde4bace7ac1fe1 # 1.5.0 diff --git a/benchmarks/java/pom.xml b/benchmarks/java/pom.xml index 9a625d7bb9..c85fde3e03 100644 --- a/benchmarks/java/pom.xml +++ b/benchmarks/java/pom.xml @@ -268,7 +268,7 @@ + name="META-INF/versions/25/org/apache/fory/reflect/FieldAccessorStrategy.class"/> + file="${jdk25.benchmark.check.dir}/META-INF/versions/25/org/apache/fory/reflect/FieldAccessorStrategy.class" + property="jdk25.benchmark.fieldaccessorstrategy.present"/> @@ -315,8 +315,8 @@ unless="jdk25.benchmark.jdkaccess.present" message="JDK25 benchmark jar is missing the versioned _JDKAccess class."/> + unless="jdk25.benchmark.fieldaccessorstrategy.present" + message="JDK25 benchmark jar is missing the versioned FieldAccessorStrategy class."/> diff --git a/benchmarks/java/src/main/java/org/apache/fory/benchmark/Jdk25MrJarCheck.java b/benchmarks/java/src/main/java/org/apache/fory/benchmark/Jdk25MrJarCheck.java index 41aae1fdae..4890268b0b 100644 --- a/benchmarks/java/src/main/java/org/apache/fory/benchmark/Jdk25MrJarCheck.java +++ b/benchmarks/java/src/main/java/org/apache/fory/benchmark/Jdk25MrJarCheck.java @@ -20,8 +20,6 @@ package org.apache.fory.benchmark; import org.apache.fory.memory.MemoryBuffer; -import org.apache.fory.platform.internal._JDKAccess; -import org.apache.fory.reflect.FieldAccessor; /** Runtime smoke check that JDK25 benchmark runs load the multi-release Fory classes. */ public final class Jdk25MrJarCheck { @@ -30,10 +28,10 @@ private Jdk25MrJarCheck() {} public static void main(String[] args) { verifyClass(MemoryBuffer.class); verifyMissing("org.apache.fory.platform.UnsafeOps"); - verifyClass(_JDKAccess.class); - verifyClass(FieldAccessor.class); + Class jdkAccess = verifyClass("org.apache.fory.platform.internal._JDKAccess"); + verifyClass("org.apache.fory.reflect.FieldAccessorStrategy"); verifyClass("org.apache.fory.serializer.PlatformStringUtils"); - if (_JDKAccess.UNSAFE != null) { + if (getUnsafeField(jdkAccess) != null) { throw new IllegalStateException("JDK25 benchmark jar loaded Unsafe-backed _JDKAccess"); } } @@ -47,9 +45,11 @@ private static void verifyMissing(String className) { } } - private static void verifyClass(String className) { + private static Class verifyClass(String className) { try { - verifyClass(Class.forName(className)); + Class cls = Class.forName(className); + verifyClass(cls); + return cls; } catch (ClassNotFoundException e) { throw new IllegalStateException("JDK25 benchmark jar is missing " + className, e); } @@ -62,4 +62,14 @@ private static void verifyClass(Class cls) { throw new IllegalStateException("JDK25 benchmark jar loaded root class for " + cls); } } + + private static Object getUnsafeField(Class jdkAccess) { + try { + return jdkAccess.getField("UNSAFE").get(null); + } catch (NoSuchFieldException expected) { + return null; + } catch (ReflectiveOperationException e) { + throw new IllegalStateException("Failed to inspect _JDKAccess.UNSAFE", e); + } + } } diff --git a/benchmarks/java/src/main/java/org/apache/fory/benchmark/MemorySuite.java b/benchmarks/java/src/main/java/org/apache/fory/benchmark/MemorySuite.java index f46b5aacc5..eea85fddad 100644 --- a/benchmarks/java/src/main/java/org/apache/fory/benchmark/MemorySuite.java +++ b/benchmarks/java/src/main/java/org/apache/fory/benchmark/MemorySuite.java @@ -32,13 +32,10 @@ import org.openjdk.jmh.annotations.Scope; import org.openjdk.jmh.annotations.Setup; import org.openjdk.jmh.annotations.State; -import sun.misc.Unsafe; @BenchmarkMode(Mode.Throughput) @CompilerControl(value = CompilerControl.Mode.INLINE) public class MemorySuite { - private static final Unsafe UNSAFE = UnsafeAccess.load(); - private static final int BYTE_ARRAY_OFFSET = UNSAFE.arrayBaseOffset(byte[].class); static int arrLen = 32; static { @@ -138,13 +135,6 @@ public Object systemArrayCopy(MemoryState state) { return target; } - @org.openjdk.jmh.annotations.Benchmark - public Object unsafeCopy(MemoryState state) { - UNSAFE.copyMemory( - state.bytes, BYTE_ARRAY_OFFSET, target, BYTE_ARRAY_OFFSET, state.bytes.length); - return target; - } - // @org.openjdk.jmh.annotations.Benchmark public Object arrayAssignCopy(MemoryState state) { byte[] bytes = state.bytes; diff --git a/benchmarks/java/src/main/java/org/apache/fory/benchmark/NewJava11StringSuite.java b/benchmarks/java/src/main/java/org/apache/fory/benchmark/NewJava11StringSuite.java index 762ec646a1..4ed79ad02b 100644 --- a/benchmarks/java/src/main/java/org/apache/fory/benchmark/NewJava11StringSuite.java +++ b/benchmarks/java/src/main/java/org/apache/fory/benchmark/NewJava11StringSuite.java @@ -19,6 +19,7 @@ package org.apache.fory.benchmark; +import java.nio.charset.StandardCharsets; import org.apache.fory.Fory; import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.platform.JdkVersion; @@ -26,26 +27,18 @@ import org.apache.fory.util.Preconditions; import org.apache.fory.util.StringUtils; import org.openjdk.jmh.Main; -import sun.misc.Unsafe; public class NewJava11StringSuite { - private static final Unsafe UNSAFE = UnsafeAccess.load(); - static String str = StringUtils.random(10); - static byte[] strBytes; - static byte coder; + static byte[] strBytes = str.getBytes(StandardCharsets.ISO_8859_1); + static byte coder = 0; static { if (JdkVersion.MAJOR_VERSION > 8) { - strBytes = (byte[]) UNSAFE.getObject(str, fieldOffset(String.class, "value")); - coder = UNSAFE.getByte(str, fieldOffset(String.class, "coder")); + Preconditions.checkArgument(new String(strBytes, StandardCharsets.ISO_8859_1).equals(str)); } } - private static final long STRING_VALUE_FIELD_OFFSET = fieldOffset(String.class, "value"); - private static final long STRING_CODER_FIELD_OFFSET = fieldOffset(String.class, "coder"); - - private static String stubStr = new String(new char[] {Character.MAX_VALUE, Character.MIN_VALUE}); private static Fory fory = Fory.builder().withStringCompressed(true).requireClassRegistration(false).build(); private static StringSerializer stringSerializer = new StringSerializer(fory.getConfig()); @@ -55,27 +48,11 @@ public class NewJava11StringSuite { stringSerializer.writeString(buffer, str); } - private static long fieldOffset(Class type, String fieldName) { - try { - return UNSAFE.objectFieldOffset(type.getDeclaredField(fieldName)); - } catch (NoSuchFieldException e) { - throw new IllegalStateException(e); - } - } - // @Benchmark public Object createJDK11StringByCopyStr() { return new String(str); } - // @Benchmark - public Object createJDK11StringByUnsafe() { - String str = new String(stubStr); - UNSAFE.putObject(str, STRING_VALUE_FIELD_OFFSET, strBytes); - UNSAFE.putByte(str, STRING_CODER_FIELD_OFFSET, coder); - return str; - } - // @Benchmark public Object createJDK8StringByMethodHandle() { return StringSerializer.newBytesStringZeroCopy(coder, strBytes); @@ -88,7 +65,6 @@ public Object createJDK8StringByFory() { } public static void main(String[] args) throws Exception { - Preconditions.checkArgument(new NewJava11StringSuite().createJDK11StringByUnsafe().equals(str)); if (args.length == 0) { String commandLine = "org.apache.fory.*NewJava11StringSuite.* -f 3 -wi 5 -i 3 -t 1 -w 2s -r 2s -rf csv"; diff --git a/benchmarks/java/src/main/java/org/apache/fory/benchmark/NewStringSuite.java b/benchmarks/java/src/main/java/org/apache/fory/benchmark/NewStringSuite.java index 43e9fe4a1f..af5a4d8942 100644 --- a/benchmarks/java/src/main/java/org/apache/fory/benchmark/NewStringSuite.java +++ b/benchmarks/java/src/main/java/org/apache/fory/benchmark/NewStringSuite.java @@ -22,11 +22,8 @@ import org.apache.fory.serializer.StringSerializer; import org.apache.fory.util.StringUtils; import org.openjdk.jmh.Main; -import sun.misc.Unsafe; public class NewStringSuite { - private static final Unsafe UNSAFE = UnsafeAccess.load(); - static String str = StringUtils.random(230); static char[] strData = str.toCharArray(); static byte[] array = new byte[strData.length * 2]; @@ -41,24 +38,6 @@ public Object createJDK8StringByCopy() { return new String(strData); } - private static final long STRING_VALUE_FIELD_OFFSET = fieldOffset(String.class, "value"); - private static String stubStr = new String(new char[] {Character.MAX_VALUE, Character.MIN_VALUE}); - - private static long fieldOffset(Class type, String fieldName) { - try { - return UNSAFE.objectFieldOffset(type.getDeclaredField(fieldName)); - } catch (NoSuchFieldException e) { - throw new IllegalStateException(e); - } - } - - // @Benchmark - public Object createJDK8StringByUnsafe() { - String str = new String(stubStr); - UNSAFE.putObject(str, STRING_VALUE_FIELD_OFFSET, strData); - return str; - } - // @Benchmark public Object createJDK8StringByMethodHandle() { return StringSerializer.newCharsStringZeroCopy(strData); diff --git a/benchmarks/java/src/main/java/org/apache/fory/benchmark/UnsafeAccess.java b/benchmarks/java/src/main/java/org/apache/fory/benchmark/UnsafeAccess.java deleted file mode 100644 index c99cc81183..0000000000 --- a/benchmarks/java/src/main/java/org/apache/fory/benchmark/UnsafeAccess.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.fory.benchmark; - -import java.lang.reflect.Field; -import sun.misc.Unsafe; - -final class UnsafeAccess { - private UnsafeAccess() {} - - static Unsafe load() { - try { - Field field = Unsafe.class.getDeclaredField("theUnsafe"); - field.setAccessible(true); - return (Unsafe) field.get(null); - } catch (ReflectiveOperationException e) { - throw new ExceptionInInitializerError(e); - } - } -} diff --git a/benchmarks/java25/README.md b/benchmarks/java25/README.md index 80e2928f42..10d923a1b5 100644 --- a/benchmarks/java25/README.md +++ b/benchmarks/java25/README.md @@ -5,11 +5,11 @@ This diagnostic JMH module compares direct-buffer scalar access paths used to re - `MemorySegment.get/set` with native-order unaligned layouts. - `MethodHandles.byteBufferViewVarHandle` over a direct `ByteBuffer`. -- `sun.misc.Unsafe` raw native-address access over the same direct `ByteBuffer`. Each benchmark invocation performs one absolute scalar access at a rolling aligned offset. This is closer to generated serializer calls into `MemoryBuffer.writeInt32`, `_unsafePutInt64`, and matching -read paths than a bulk array-copy benchmark. +read paths than a bulk array-copy benchmark, while keeping the JDK25 benchmark module free of +`jdk.unsupported` APIs. Build and run with JDK 25: @@ -28,17 +28,3 @@ java -jar target/java25-memory-access-benchmarks.jar \ 'org.apache.fory.benchmark.java25.DirectToHeapCopyBenchmark.*' \ -f 1 -wi 5 -i 5 -t 1 -w 1s -r 1s ``` - -The benchmark class adds fork JVM options only for the explicit Unsafe baseline: - -```text ---add-opens=java.base/java.nio=ALL-UNNAMED ---sun-misc-unsafe-memory-access=allow -``` - -These flags are not part of Fory's JDK25 zero-Unsafe deployment contract. They are present here so -the benchmark can compare supported `ByteBuffer`/FFM access with the old raw-address Unsafe -baseline in the same process shape. - -If you run with `-f 0`, pass those options to the outer `java` command because JMH will not fork a -child JVM. diff --git a/benchmarks/java25/src/main/java/org/apache/fory/benchmark/java25/DirectMemoryAccessBenchmark.java b/benchmarks/java25/src/main/java/org/apache/fory/benchmark/java25/DirectMemoryAccessBenchmark.java index da251382cf..a66e4a86aa 100644 --- a/benchmarks/java25/src/main/java/org/apache/fory/benchmark/java25/DirectMemoryAccessBenchmark.java +++ b/benchmarks/java25/src/main/java/org/apache/fory/benchmark/java25/DirectMemoryAccessBenchmark.java @@ -23,8 +23,6 @@ import java.lang.foreign.ValueLayout; import java.lang.invoke.MethodHandles; import java.lang.invoke.VarHandle; -import java.lang.reflect.Field; -import java.nio.Buffer; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.util.concurrent.TimeUnit; @@ -39,18 +37,12 @@ import org.openjdk.jmh.annotations.State; import org.openjdk.jmh.annotations.Threads; import org.openjdk.jmh.annotations.Warmup; -import sun.misc.Unsafe; @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.NANOSECONDS) @Warmup(iterations = 5, time = 1) @Measurement(iterations = 5, time = 1) -@Fork( - value = 1, - jvmArgsAppend = { - "--add-opens=java.base/java.nio=ALL-UNNAMED", - "--sun-misc-unsafe-memory-access=allow" - }) +@Fork(1) @Threads(1) public class DirectMemoryAccessBenchmark { private static final int BUFFER_BYTES = 64 * 1024; @@ -59,8 +51,6 @@ public class DirectMemoryAccessBenchmark { private static final int INT_SLOT_MASK = INT_SLOTS - 1; private static final int LONG_SLOT_MASK = LONG_SLOTS - 1; private static final ByteOrder NATIVE_ORDER = ByteOrder.nativeOrder(); - private static final Unsafe UNSAFE = loadUnsafe(); - private static final long BUFFER_ADDRESS_OFFSET = bufferAddressOffset(); private static final VarHandle INT_HANDLE = MethodHandles.byteBufferViewVarHandle(int[].class, NATIVE_ORDER); private static final VarHandle LONG_HANDLE = @@ -74,7 +64,6 @@ public class DirectMemoryAccessBenchmark { public static class DirectState { ByteBuffer buffer; MemorySegment segment; - long address; int intValue; int intCursor; long longValue; @@ -84,18 +73,17 @@ public static class DirectState { public void setup() { buffer = ByteBuffer.allocateDirect(BUFFER_BYTES).order(NATIVE_ORDER); segment = MemorySegment.ofBuffer(buffer); - address = UNSAFE.getLong(buffer, BUFFER_ADDRESS_OFFSET); intValue = 0x12345678; longValue = 0x123456789abcdef0L; for (int i = 0; i < INT_SLOTS; i++) { int intOffset = i << 2; int intPattern = intValue + i; - UNSAFE.putInt(null, address + intOffset, intPattern); + segment.set(INT_LAYOUT, intOffset, intPattern); } for (int i = 0; i < LONG_SLOTS; i++) { int longOffset = i << 3; long longPattern = longValue + i; - UNSAFE.putLong(null, address + longOffset, longPattern); + segment.set(LONG_LAYOUT, longOffset, longPattern); } } @@ -134,14 +122,6 @@ public int varHandlePutInt(DirectState state) { return value; } - @Benchmark - public int unsafePutInt(DirectState state) { - int offset = state.nextIntOffset(); - int value = state.nextIntValue(); - UNSAFE.putInt(null, state.address + offset, value); - return value; - } - @Benchmark public int memorySegmentGetInt(DirectState state) { return state.segment.get(INT_LAYOUT, state.nextIntOffset()); @@ -152,11 +132,6 @@ public int varHandleGetInt(DirectState state) { return (int) INT_HANDLE.get(state.buffer, state.nextIntOffset()); } - @Benchmark - public int unsafeGetInt(DirectState state) { - return UNSAFE.getInt(null, state.address + state.nextIntOffset()); - } - @Benchmark public long memorySegmentPutLong(DirectState state) { int offset = state.nextLongOffset(); @@ -173,14 +148,6 @@ public long varHandlePutLong(DirectState state) { return value; } - @Benchmark - public long unsafePutLong(DirectState state) { - int offset = state.nextLongOffset(); - long value = state.nextLongValue(); - UNSAFE.putLong(null, state.address + offset, value); - return value; - } - @Benchmark public long memorySegmentGetLong(DirectState state) { return state.segment.get(LONG_LAYOUT, state.nextLongOffset()); @@ -190,27 +157,4 @@ public long memorySegmentGetLong(DirectState state) { public long varHandleGetLong(DirectState state) { return (long) LONG_HANDLE.get(state.buffer, state.nextLongOffset()); } - - @Benchmark - public long unsafeGetLong(DirectState state) { - return UNSAFE.getLong(null, state.address + state.nextLongOffset()); - } - - private static Unsafe loadUnsafe() { - try { - Field field = Unsafe.class.getDeclaredField("theUnsafe"); - field.setAccessible(true); - return (Unsafe) field.get(null); - } catch (ReflectiveOperationException e) { - throw new ExceptionInInitializerError(e); - } - } - - private static long bufferAddressOffset() { - try { - return UNSAFE.objectFieldOffset(Buffer.class.getDeclaredField("address")); - } catch (NoSuchFieldException e) { - throw new ExceptionInInitializerError(e); - } - } } diff --git a/benchmarks/java25/src/main/java/org/apache/fory/benchmark/java25/DirectToHeapCopyBenchmark.java b/benchmarks/java25/src/main/java/org/apache/fory/benchmark/java25/DirectToHeapCopyBenchmark.java index f3c3658471..5b64c1b611 100644 --- a/benchmarks/java25/src/main/java/org/apache/fory/benchmark/java25/DirectToHeapCopyBenchmark.java +++ b/benchmarks/java25/src/main/java/org/apache/fory/benchmark/java25/DirectToHeapCopyBenchmark.java @@ -20,8 +20,6 @@ package org.apache.fory.benchmark.java25; import java.lang.foreign.MemorySegment; -import java.lang.reflect.Field; -import java.nio.Buffer; import java.nio.ByteBuffer; import java.util.concurrent.TimeUnit; import org.openjdk.jmh.annotations.Benchmark; @@ -36,24 +34,15 @@ import org.openjdk.jmh.annotations.State; import org.openjdk.jmh.annotations.Threads; import org.openjdk.jmh.annotations.Warmup; -import sun.misc.Unsafe; @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.NANOSECONDS) @Warmup(iterations = 5, time = 1) @Measurement(iterations = 5, time = 1) -@Fork( - value = 1, - jvmArgsAppend = { - "--add-opens=java.base/java.nio=ALL-UNNAMED", - "--sun-misc-unsafe-memory-access=allow" - }) +@Fork(1) @Threads(1) public class DirectToHeapCopyBenchmark { private static final int BUFFER_BYTES = 64 * 1024; - private static final Unsafe UNSAFE = loadUnsafe(); - private static final long BUFFER_ADDRESS_OFFSET = bufferAddressOffset(); - private static final int BYTE_ARRAY_OFFSET = UNSAFE.arrayBaseOffset(byte[].class); @State(Scope.Thread) public static class CopyState { @@ -64,7 +53,6 @@ public static class CopyState { MemorySegment directSegment; byte[] heapBuffer; MemorySegment heapSegment; - long directAddress; @Setup public void setup() { @@ -72,9 +60,8 @@ public void setup() { directSegment = MemorySegment.ofBuffer(directBuffer); heapBuffer = new byte[BUFFER_BYTES]; heapSegment = MemorySegment.ofArray(heapBuffer); - directAddress = UNSAFE.getLong(directBuffer, BUFFER_ADDRESS_OFFSET); for (int i = 0; i < BUFFER_BYTES; i++) { - UNSAFE.putByte(null, directAddress + i, (byte) (i * 31)); + directBuffer.put(i, (byte) (i * 31)); } } } @@ -94,30 +81,4 @@ public int memorySegmentCopy(CopyState state) { MemorySegment.copy(state.directSegment, 0, state.heapSegment, 0, copySize); return heap[copySize - 1]; } - - @Benchmark - public int unsafeCopyMemory(CopyState state) { - int copySize = state.copySize; - byte[] heap = state.heapBuffer; - UNSAFE.copyMemory(null, state.directAddress, heap, BYTE_ARRAY_OFFSET, copySize); - return heap[copySize - 1]; - } - - private static Unsafe loadUnsafe() { - try { - Field field = Unsafe.class.getDeclaredField("theUnsafe"); - field.setAccessible(true); - return (Unsafe) field.get(null); - } catch (ReflectiveOperationException e) { - throw new ExceptionInInitializerError(e); - } - } - - private static long bufferAddressOffset() { - try { - return UNSAFE.objectFieldOffset(Buffer.class.getDeclaredField("address")); - } catch (NoSuchFieldException e) { - throw new ExceptionInInitializerError(e); - } - } } diff --git a/ci/run_ci.sh b/ci/run_ci.sh index cf8beb52ca..ca52828ffd 100755 --- a/ci/run_ci.sh +++ b/ci/run_ci.sh @@ -102,19 +102,19 @@ graalvm_test() { } jdk25_access_options() { - local fory_open_targets="${1:-ALL-UNNAMED}" + local fory_open_targets="${1:-org.apache.fory.core}" printf "%s" "--sun-misc-unsafe-memory-access=deny" printf " %s" "--add-opens=java.base/java.lang.invoke=${fory_open_targets}" } jdk26_final_field_options() { - local fory_modules="${1:-ALL-UNNAMED}" + local fory_modules="${1:-org.apache.fory.core}" printf "%s" "--enable-final-field-mutation=${fory_modules}" } jdk25_plus_options() { local java_major="$1" - local fory_targets="${2:-ALL-UNNAMED}" + local fory_targets="${2:-org.apache.fory.core}" printf "%s" "$(jdk25_access_options "$fory_targets")" if [[ "$java_major" -ge 26 ]]; then printf " %s" "$(jdk26_final_field_options "$fory_targets")" @@ -198,11 +198,19 @@ integration_tests() { cd "$ROOT"/integration_tests/jdk_compatibility_tests mvn -T10 -B --no-transfer-progress clean test for jdk in "${JDKS[@]}"; do + if [[ "$jdk" =~ zulu([0-9]+) && "${BASH_REMATCH[1]}" -ge 25 ]]; then + echo "Skipping classpath JDK compatibility data generation for ${jdk}; JDK25+ zero-Unsafe coverage runs on JPMS" + continue + fi use_jdk "$jdk" echo "First round for generate data: ${jdk}" mvn -T10 --no-transfer-progress clean test -Dtest=org.apache.fory.integration_tests.JDKCompatibilityTest done for jdk in "${JDKS[@]}"; do + if [[ "$jdk" =~ zulu([0-9]+) && "${BASH_REMATCH[1]}" -ge 25 ]]; then + echo "Skipping classpath JDK compatibility verification for ${jdk}; JDK25+ zero-Unsafe coverage runs on JPMS" + continue + fi use_jdk "$jdk" echo "Second round for compatibility: ${jdk}" mvn -T10 --no-transfer-progress clean test -Dtest=org.apache.fory.integration_tests.JDKCompatibilityTest @@ -239,13 +247,6 @@ jdk17_plus_tests() { fi if [[ "$java_major" -ge 25 ]]; then unset JDK_JAVA_OPTIONS - echo "Executing JDK${java_major} packaged classpath tests" - cd "$ROOT/integration_tests/jdk_compatibility_tests" - mvn -T10 --batch-mode --no-transfer-progress clean test - testcode=$? - if [[ $testcode -ne 0 ]]; then - exit $testcode - fi echo "Executing JDK${java_major} JPMS tests" cd "$ROOT/integration_tests/jpms_tests" mvn -T10 --batch-mode --no-transfer-progress clean test diff --git a/ci/tasks/java.py b/ci/tasks/java.py index 7784e1b488..b35946bfac 100644 --- a/ci/tasks/java.py +++ b/ci/tasks/java.py @@ -77,18 +77,18 @@ def install_jdks(): logging.info("JDKs downloaded and installed successfully") -def jdk25_access_options(fory_targets="ALL-UNNAMED"): +def jdk25_access_options(fory_targets="org.apache.fory.core"): return [ "--sun-misc-unsafe-memory-access=deny", f"--add-opens=java.base/java.lang.invoke={fory_targets}", ] -def jdk26_final_field_options(fory_targets="ALL-UNNAMED"): +def jdk26_final_field_options(fory_targets="org.apache.fory.core"): return [f"--enable-final-field-mutation={fory_targets}"] -def jdk25_plus_options(java_version, fory_targets="ALL-UNNAMED"): +def jdk25_plus_options(java_version, fory_targets="org.apache.fory.core"): options = jdk25_access_options(fory_targets) if int(java_version) >= 26: options.extend(jdk26_final_field_options(fory_targets)) @@ -272,9 +272,6 @@ def run_jdk17_plus(java_version="17"): "-DskipTests -Dmaven.compiler.parameters=true" ) os.environ.pop("JDK_JAVA_OPTIONS", None) - logging.info(f"Executing JDK{java_version} packaged classpath tests") - common.cd_project_subdir("integration_tests/jdk_compatibility_tests") - common.exec_cmd("mvn -T10 --batch-mode --no-transfer-progress clean test") logging.info(f"Executing JDK{java_version} JPMS tests") common.cd_project_subdir("integration_tests/jpms_tests") common.exec_cmd("mvn -T10 --batch-mode --no-transfer-progress clean test") @@ -332,6 +329,12 @@ def run_integration_tests(): # First round: Generate serialized data files logging.info("First round: Generate serialized data files for each JDK version") for java_version, jdk in JDKS.items(): + if int(java_version) >= 25: + logging.info( + "Skipping classpath JDK compatibility data generation for " + f"JDK{java_version}; JDK25+ zero-Unsafe coverage runs on JPMS" + ) + continue use_jdk(java_version) logging.info(f"Generating data with JDK: {jdk}") @@ -342,6 +345,12 @@ def run_integration_tests(): # Second round: Test cross-JDK compatibility logging.info("Second round: Test cross-JDK compatibility") for java_version, jdk in JDKS.items(): + if int(java_version) >= 25: + logging.info( + "Skipping classpath JDK compatibility verification for " + f"JDK{java_version}; JDK25+ zero-Unsafe coverage runs on JPMS" + ) + continue use_jdk(java_version) logging.info(f"Testing compatibility with JDK: {jdk}") @@ -387,15 +396,23 @@ def run_release(): ) common.cd_project_subdir("java") - # Clean and install without tests first - logging.info("Cleaning and installing dependencies") - common.exec_cmd("mvn -T10 -B --no-transfer-progress clean install -DskipTests") + previous_jdk_options = os.environ.get("JDK_JAVA_OPTIONS") + os.environ["JDK_JAVA_OPTIONS"] = " ".join(jdk25_javac_options()) + try: + # Clean and install without tests first + logging.info("Cleaning and installing dependencies") + common.exec_cmd("mvn -T10 -B --no-transfer-progress clean install -DskipTests") - # Deploy to Maven Central - logging.info("Deploying to Maven Central") - common.exec_cmd( - "mvn -T10 -B --no-transfer-progress clean deploy -Dgpg.skip -DskipTests -Papache-release" - ) + # Deploy to Maven Central + logging.info("Deploying to Maven Central") + common.exec_cmd( + "mvn -T10 -B --no-transfer-progress clean deploy -Dgpg.skip -DskipTests -Papache-release" + ) + finally: + if previous_jdk_options is None: + os.environ.pop("JDK_JAVA_OPTIONS", None) + else: + os.environ["JDK_JAVA_OPTIONS"] = previous_jdk_options logging.info("Release to Maven Central completed successfully") diff --git a/docs/guide/java/native-serialization.md b/docs/guide/java/native-serialization.md index d13d73b8d4..58e1392dff 100644 --- a/docs/guide/java/native-serialization.md +++ b/docs/guide/java/native-serialization.md @@ -201,8 +201,8 @@ object creation and field setting. On JDK25+ with Unsafe memory access denied, F if the class cannot be created by supported Java mechanisms. Use `@ForyConstructor`, `registerConstructor(...)`, a record canonical constructor, or a custom serializer for those classes. Use the `java.base/java.lang.invoke` open shown in troubleshooting for supported JDK25+ access paths. -On JDK26+, enable final-field mutation for the Fory runtime module, or for `ALL-UNNAMED` when Fory is -loaded from the classpath. JDK25 does not have the final-field mutation flag. See +On JDK26+, enable final-field mutation for the Fory runtime module. JDK25 does not have the final-field +mutation flag. See [Troubleshooting](troubleshooting.md#jdk25-zero-unsafe-mode-and-module-opens) for the required JVM flags. diff --git a/docs/guide/java/troubleshooting.md b/docs/guide/java/troubleshooting.md index 9dd2e345cb..dbbb2e509b 100644 --- a/docs/guide/java/troubleshooting.md +++ b/docs/guide/java/troubleshooting.md @@ -157,14 +157,7 @@ memory access becomes the default, start the JVM with: --sun-misc-unsafe-memory-access=deny ``` -If Fory runs from the classpath, including a modular Fory jar placed on the classpath, open -`java.base/java.lang.invoke` to the unnamed module: - -```bash ---add-opens=java.base/java.lang.invoke=ALL-UNNAMED -``` - -If Fory runs as named modules on the module path, open `java.base/java.lang.invoke` to the Fory core +Run Fory as named modules on the module path and open `java.base/java.lang.invoke` to the Fory core module: ```bash @@ -181,12 +174,6 @@ Fory module name on the module path: --enable-final-field-mutation=org.apache.fory.core ``` -Use `ALL-UNNAMED` when running Fory on the classpath: - -```bash ---enable-final-field-mutation=ALL-UNNAMED -``` - Fory can restore those final fields when final-field mutation is enabled. JDK25 has no `--enable-final-field-mutation` option, so no final-field mutation flag is needed on JDK25. Named application modules that contain private fields still need to open the application package to diff --git a/docs/guide/kotlin/configuration.md b/docs/guide/kotlin/configuration.md index 8f338e529e..a2fcaebce3 100644 --- a/docs/guide/kotlin/configuration.md +++ b/docs/guide/kotlin/configuration.md @@ -100,8 +100,21 @@ All configuration options from Fory Java are available. See [Java Configuration] ## JDK25+ Zero-Unsafe Mode On JDK25+ with Unsafe memory access denied, Kotlin classes with final constructor properties need -bindable constructor metadata so Fory can call the primary constructor instead of allocating an -uninitialized instance. Enable Java parameter metadata for Kotlin compilation: +an explicit constructor mapping when Fory must call the primary constructor. Annotate the +constructor with `@ForyConstructor`, register the constructor with `registerConstructor(...)`, use a +generated serializer that carries explicit constructor metadata, or provide a custom serializer. + +```kotlin +import org.apache.fory.annotation.ForyConstructor + +class User @ForyConstructor("name", "age") constructor( + val name: String, + val age: Int, +) +``` + +For generated serializers and tooling that read Java parameter metadata, enable Java parameter +metadata for Kotlin compilation: ```kotlin kotlin { diff --git a/integration_tests/graalvm_tests/pom.xml b/integration_tests/graalvm_tests/pom.xml index acb6ebb4da..f2f9945dd8 100644 --- a/integration_tests/graalvm_tests/pom.xml +++ b/integration_tests/graalvm_tests/pom.xml @@ -176,7 +176,6 @@ -H:+UnlockExperimentalVMOptions - -J--add-opens=java.base/java.lang.invoke=ALL-UNNAMED @@ -188,17 +187,13 @@ java-agent - exec + java test - ${java.home}/bin/java - - --add-opens=java.base/java.lang.invoke=ALL-UNNAMED - -classpath - - ${mainClass} - + ${mainClass} + true + false @@ -206,27 +201,5 @@ - - jdk25-native - - [25,) - - - - - org.graalvm.buildtools - native-maven-plugin - - - --sun-misc-unsafe-memory-access=deny - - - -J--sun-misc-unsafe-memory-access=deny - - - - - - diff --git a/integration_tests/graalvm_tests/src/main/java/org/apache/fory/graalvm/FeatureTestExample.java b/integration_tests/graalvm_tests/src/main/java/org/apache/fory/graalvm/FeatureTestExample.java index d81ac3125d..43a202d170 100644 --- a/integration_tests/graalvm_tests/src/main/java/org/apache/fory/graalvm/FeatureTestExample.java +++ b/integration_tests/graalvm_tests/src/main/java/org/apache/fory/graalvm/FeatureTestExample.java @@ -19,11 +19,11 @@ package org.apache.fory.graalvm; -import java.beans.ConstructorProperties; import java.lang.reflect.InvocationHandler; import java.lang.reflect.Method; import java.lang.reflect.Proxy; import org.apache.fory.Fory; +import org.apache.fory.annotation.ForyConstructor; import org.apache.fory.builder.Generated; import org.apache.fory.platform.GraalvmSupport; import org.apache.fory.util.Preconditions; @@ -51,7 +51,7 @@ public interface TestInterface { public static class TestInvocationHandler implements InvocationHandler { private final String value; - @ConstructorProperties("value") + @ForyConstructor("value") public TestInvocationHandler(String value) { this.value = value; } diff --git a/integration_tests/graalvm_tests/src/main/java/org/apache/fory/graalvm/Foo.java b/integration_tests/graalvm_tests/src/main/java/org/apache/fory/graalvm/Foo.java index e5d4a0065f..d4ad4a6b8b 100644 --- a/integration_tests/graalvm_tests/src/main/java/org/apache/fory/graalvm/Foo.java +++ b/integration_tests/graalvm_tests/src/main/java/org/apache/fory/graalvm/Foo.java @@ -19,11 +19,11 @@ package org.apache.fory.graalvm; -import java.beans.ConstructorProperties; import java.io.Serializable; import java.util.List; import java.util.Map; import java.util.Objects; +import org.apache.fory.annotation.ForyConstructor; public class Foo implements Serializable { int f1; @@ -31,7 +31,7 @@ public class Foo implements Serializable { List f3; Map f4; - @ConstructorProperties({"f1", "f2", "f3", "f4"}) + @ForyConstructor({"f1", "f2", "f3", "f4"}) public Foo(int f1, String f2, List f3, Map f4) { this.f1 = f1; this.f2 = f2; diff --git a/integration_tests/graalvm_tests/src/main/java/org/apache/fory/graalvm/ObjectStreamExample.java b/integration_tests/graalvm_tests/src/main/java/org/apache/fory/graalvm/ObjectStreamExample.java index 68dca1b20b..c7978eb8e1 100644 --- a/integration_tests/graalvm_tests/src/main/java/org/apache/fory/graalvm/ObjectStreamExample.java +++ b/integration_tests/graalvm_tests/src/main/java/org/apache/fory/graalvm/ObjectStreamExample.java @@ -19,7 +19,6 @@ package org.apache.fory.graalvm; -import java.beans.ConstructorProperties; import java.io.Serializable; import java.util.AbstractMap; import java.util.Arrays; @@ -28,6 +27,7 @@ import java.util.TreeMap; import java.util.TreeSet; import org.apache.fory.Fory; +import org.apache.fory.annotation.ForyConstructor; import org.apache.fory.serializer.ObjectStreamSerializer; import org.apache.fory.serializer.collection.CollectionSerializers; import org.apache.fory.serializer.collection.MapSerializers; @@ -73,7 +73,7 @@ public ObjectStreamExample() { this(new int[10]); } - @ConstructorProperties("ints") + @ForyConstructor("ints") public ObjectStreamExample(int[] ints) { this.ints = ints; } diff --git a/integration_tests/jdk_compatibility_tests/pom.xml b/integration_tests/jdk_compatibility_tests/pom.xml index d5df74e988..08a1257b24 100644 --- a/integration_tests/jdk_compatibility_tests/pom.xml +++ b/integration_tests/jdk_compatibility_tests/pom.xml @@ -35,7 +35,6 @@ 8 8 UTF-8 - @@ -83,8 +82,11 @@ + - jdk25-and-higher + skip-jdk25-and-higher [25,) @@ -94,27 +96,12 @@ org.apache.maven.plugins maven-surefire-plugin - - --sun-misc-unsafe-memory-access=deny - --add-opens=java.base/java.lang.invoke=ALL-UNNAMED - ${fory.final.field.mutation.arg} - + true - - jdk26-and-higher - - [26,) - - - - --enable-final-field-mutation=ALL-UNNAMED - - - diff --git a/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/StaticSerializerSourceWriter.java b/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/StaticSerializerSourceWriter.java index 7eabcda6b2..596b43113a 100644 --- a/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/StaticSerializerSourceWriter.java +++ b/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/StaticSerializerSourceWriter.java @@ -90,6 +90,7 @@ private void writeClassStart() { builder.append(" private final SerializationFieldInfo[] fieldsById;\n"); builder.append(" private final int[] constructorFieldIds;\n"); builder.append(" private final long[] constructorFieldBits;\n"); + builder.append(" private final Class[] constructorFieldTypes;\n"); builder.append(" private final int classVersionHash;\n"); builder.append(" private final boolean sameSchemaCompatible;\n\n"); } @@ -144,6 +145,7 @@ private void writeConstructors() { builder.append(" this.fieldsById = null;\n"); builder.append(" this.constructorFieldIds = null;\n"); builder.append(" this.constructorFieldBits = null;\n"); + builder.append(" this.constructorFieldTypes = null;\n"); builder.append(" this.classVersionHash = 0;\n"); builder.append(" this.sameSchemaCompatible = false;\n"); builder.append(" }\n\n"); @@ -179,6 +181,8 @@ private void writeConstructorBody(String fieldGroupsExpression, String sameSchem " this.constructorFieldIds = objectCreator.hasConstructorFields() ? buildConstructorFieldIds(DESCRIPTORS) : null;\n"); builder.append( " this.constructorFieldBits = buildConstructorFieldBits(DESCRIPTORS.size(), constructorFieldIds);\n"); + builder.append( + " this.constructorFieldTypes = constructorFieldIds != null ? constructorFieldTypes() : null;\n"); builder.append( " this.classVersionHash = typeResolver.checkClassVersion() ? computeClassVersionHash(DESCRIPTORS) : 0;\n"); builder.append(" this.sameSchemaCompatible = ").append(sameSchemaExpression).append(";\n"); @@ -487,7 +491,7 @@ private void writeConstructorRead() { .append(" return (") .append(struct.typeName) .append( - ") objectCreator.newInstanceWithArguments(constructorArgs(fieldValues, constructorFieldIds, objectCreator.getConstructorFieldTypes()));\n"); + ") objectCreator.newInstanceWithArguments(constructorArgs(fieldValues, constructorFieldIds, constructorFieldTypes));\n"); builder.append(" }\n\n"); builder diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/AbstractObjectSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/AbstractObjectSerializer.java index 05460d0b56..520070ccd3 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/AbstractObjectSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/AbstractObjectSerializer.java @@ -73,6 +73,10 @@ public abstract class AbstractObjectSerializer extends Serializer { protected final TypeResolver typeResolver; protected final boolean isRecord; protected final ObjectCreator objectCreator; + private final String[] objectCreatorConstructorFieldNames; + private final Class[] objectCreatorConstructorFieldDeclaringClasses; + private final Class[] objectCreatorConstructorFieldTypes; + private final boolean[] objectCreatorConstructorFieldFinal; private SerializationFieldInfo[] fieldInfos; private RecordInfo copyRecordInfo; @@ -82,6 +86,10 @@ protected AbstractObjectSerializer() { this.typeResolver = null; this.isRecord = false; this.objectCreator = null; + this.objectCreatorConstructorFieldNames = null; + this.objectCreatorConstructorFieldDeclaringClasses = null; + this.objectCreatorConstructorFieldTypes = null; + this.objectCreatorConstructorFieldFinal = null; } public AbstractObjectSerializer(TypeResolver typeResolver, Class type) { @@ -95,6 +103,18 @@ public AbstractObjectSerializer( this.typeResolver = typeResolver; this.isRecord = RecordUtils.isRecord(type); this.objectCreator = objectCreator; + if (objectCreator.hasConstructorFields()) { + this.objectCreatorConstructorFieldNames = objectCreator.getConstructorFieldNames(); + this.objectCreatorConstructorFieldDeclaringClasses = + objectCreator.getConstructorFieldDeclaringClasses(); + this.objectCreatorConstructorFieldTypes = objectCreator.getConstructorFieldTypes(); + this.objectCreatorConstructorFieldFinal = objectCreator.getConstructorFieldFinal(); + } else { + this.objectCreatorConstructorFieldNames = null; + this.objectCreatorConstructorFieldDeclaringClasses = null; + this.objectCreatorConstructorFieldTypes = null; + this.objectCreatorConstructorFieldFinal = null; + } } static void writeField( @@ -909,7 +929,7 @@ private T copyConstructorObject(CopyContext copyContext, T originObj) { T newObj = objectCreator.newInstanceWithArguments( constructorArgs( - fieldValues, constructorFieldIndexes, objectCreator.getConstructorFieldTypes())); + fieldValues, constructorFieldIndexes, objectCreatorConstructorFieldTypes)); copyContext.reference(originObj, newObj); copyFields(copyContext, fieldInfos, originObj, newObj, constructorFieldMask, false); return newObj; @@ -1164,6 +1184,22 @@ protected T newBean() { return objectCreator.newInstance(); } + protected final String[] constructorFieldNames() { + return objectCreatorConstructorFieldNames; + } + + protected final Class[] constructorFieldDeclaringClasses() { + return objectCreatorConstructorFieldDeclaringClasses; + } + + protected final Class[] constructorFieldTypes() { + return objectCreatorConstructorFieldTypes; + } + + protected final boolean[] constructorFieldFinal() { + return objectCreatorConstructorFieldFinal; + } + protected final int[] buildConstructorFieldIndexes(SerializationFieldInfo[] fieldInfos) { return buildConstructorFieldIndexes(fieldInfos, true); } @@ -1183,12 +1219,12 @@ protected final int[] buildConstructorFieldIndexes( boolean allowMissingNonFinal, String[] defaultFields, Class[] defaultDeclaringClasses) { - String[] fieldNames = objectCreator.getConstructorFieldNames(); + String[] fieldNames = objectCreatorConstructorFieldNames; if (fieldNames.length == 0) { return null; } - Class[] declaringClasses = objectCreator.getConstructorFieldDeclaringClasses(); - boolean[] finalFields = objectCreator.getConstructorFieldFinal(); + Class[] declaringClasses = objectCreatorConstructorFieldDeclaringClasses; + boolean[] finalFields = objectCreatorConstructorFieldFinal; int[] indexes = new int[fieldNames.length]; for (int i = 0; i < fieldNames.length; i++) { Class declaringClass = declaringClasses == null ? null : declaringClasses[i]; diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/CompatibleSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/CompatibleSerializer.java index 44db7e6b86..f734b6980d 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/CompatibleSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/CompatibleSerializer.java @@ -70,6 +70,9 @@ public class CompatibleSerializer extends AbstractObjectSerializer { private final SerializationFieldInfo[] allFields; private final int[] constructorFieldIndexes; private final boolean[] constructorFieldMask; + private final String[] constructorFieldNames; + private final Class[] constructorFieldDeclaringClasses; + private final Class[] constructorFieldTypes; private final CompatibleCollectionArrayReader.ReadAction[] allCompatibleReadActions; private final boolean hasCompatibleCollectionArrayRead; private final RecordInfo recordInfo; @@ -149,9 +152,15 @@ public CompatibleSerializer(TypeResolver typeResolver, Class type, TypeDef ty defaultFieldNames(defaultValueFields), defaultDeclaringClasses(defaultValueFields)); constructorFieldMask = buildConstructorFieldMask(allFields.length, constructorFieldIndexes); + constructorFieldNames = constructorFieldNames(); + constructorFieldDeclaringClasses = constructorFieldDeclaringClasses(); + constructorFieldTypes = constructorFieldTypes(); } else { constructorFieldIndexes = null; constructorFieldMask = null; + constructorFieldNames = null; + constructorFieldDeclaringClasses = null; + constructorFieldTypes = null; } } @@ -273,17 +282,17 @@ private T newInstance() { } private Object[] compatibleConstructorArgs(Object[] fieldValues) { - String[] fieldNames = objectCreator.getConstructorFieldNames(); - Class[] declaringClasses = objectCreator.getConstructorFieldDeclaringClasses(); - Class[] fieldTypes = objectCreator.getConstructorFieldTypes(); Object[] args = new Object[constructorFieldIndexes.length]; for (int i = 0; i < constructorFieldIndexes.length; i++) { int index = constructorFieldIndexes[i]; if (index >= 0) { args[i] = fieldValues[index]; } else { - Class declaringClass = declaringClasses == null ? null : declaringClasses[i]; - args[i] = defaultConstructorValue(fieldNames[i], declaringClass, fieldTypes[i]); + Class declaringClass = + constructorFieldDeclaringClasses == null ? null : constructorFieldDeclaringClasses[i]; + args[i] = + defaultConstructorValue( + constructorFieldNames[i], declaringClass, constructorFieldTypes[i]); } } return args; @@ -396,10 +405,7 @@ private T createConstructorObject(Object[] fieldValues) { private void setNonConstructorDefaultValues(T targetObject) { DefaultValueUtils.setDefaultValues( - targetObject, - defaultValueFields, - objectCreator.getConstructorFieldNames(), - objectCreator.getConstructorFieldDeclaringClasses()); + targetObject, defaultValueFields, constructorFieldNames, constructorFieldDeclaringClasses); } private void setBufferedNonConstructorFields( @@ -429,17 +435,6 @@ private void readFields(ReadContext readContext, T targetObject) { } } - private void readFields(ReadContext readContext, T targetObject, boolean constructorFields) { - MemoryBuffer buffer = readContext.getBuffer(); - RefReader refReader = readContext.getRefReader(); - Generics generics = readContext.getGenerics(); - for (int i = 0; i < allFields.length; i++) { - if (constructorFieldMask[i] == constructorFields) { - readField(readContext, targetObject, refReader, generics, allFields[i], buffer, null); - } - } - } - private void readFields(ReadContext readContext, Object[] fields) { MemoryBuffer buffer = readContext.getBuffer(); int counter = 0; @@ -450,17 +445,6 @@ private void readFields(ReadContext readContext, Object[] fields) { } } - private void readFields(ReadContext readContext, Object[] fields, boolean constructorFields) { - MemoryBuffer buffer = readContext.getBuffer(); - RefReader refReader = readContext.getRefReader(); - Generics generics = readContext.getGenerics(); - for (int i = 0; i < allFields.length; i++) { - if (constructorFieldMask[i] == constructorFields) { - fields[i] = readField(readContext, refReader, generics, allFields[i], buffer, null); - } - } - } - private void compatibleRead( ReadContext readContext, SerializationFieldInfo fieldInfo, Object obj) { MemoryBuffer buffer = readContext.getBuffer(); @@ -485,25 +469,6 @@ private void readFieldsWithCompatibleCollectionArray(ReadContext readContext, T } } - private void readFieldsWithCompatibleCollectionArray( - ReadContext readContext, T targetObject, boolean constructorFields) { - MemoryBuffer buffer = readContext.getBuffer(); - RefReader refReader = readContext.getRefReader(); - Generics generics = readContext.getGenerics(); - for (int i = 0; i < allFields.length; i++) { - if (constructorFieldMask[i] != constructorFields) { - continue; - } - SerializationFieldInfo fieldInfo = allFields[i]; - CompatibleCollectionArrayReader.ReadAction action = - compatibleCollectionArrayReadAction(allCompatibleReadActions, i); - if (Utils.DEBUG_OUTPUT_VERBOSE) { - printFieldDebugInfo(fieldInfo, buffer); - } - readField(readContext, targetObject, refReader, generics, fieldInfo, buffer, action); - } - } - private void readFieldsWithCompatibleCollectionArray(ReadContext readContext, Object[] fields) { MemoryBuffer buffer = readContext.getBuffer(); int counter = 0; @@ -520,25 +485,6 @@ private void readFieldsWithCompatibleCollectionArray(ReadContext readContext, Ob } } - private void readFieldsWithCompatibleCollectionArray( - ReadContext readContext, Object[] fields, boolean constructorFields) { - MemoryBuffer buffer = readContext.getBuffer(); - RefReader refReader = readContext.getRefReader(); - Generics generics = readContext.getGenerics(); - for (int i = 0; i < allFields.length; i++) { - if (constructorFieldMask[i] != constructorFields) { - continue; - } - SerializationFieldInfo fieldInfo = allFields[i]; - CompatibleCollectionArrayReader.ReadAction action = - compatibleCollectionArrayReadAction(allCompatibleReadActions, i); - if (Utils.DEBUG_OUTPUT_ENABLED) { - printFieldDebugInfo(fieldInfo, buffer); - } - fields[i] = readField(readContext, refReader, generics, fieldInfo, buffer, action); - } - } - private void readField( ReadContext readContext, T targetObject, diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectSerializer.java index eb84189eec..74e45e0029 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectSerializer.java @@ -66,6 +66,7 @@ public final class ObjectSerializer extends AbstractObjectSerializer { private final SerializationFieldInfo[] allFields; private final int[] constructorFieldIndexes; private final boolean[] constructorFieldMask; + private final Class[] constructorFieldTypes; private final int classVersionHash; public ObjectSerializer(TypeResolver typeResolver, Class cls) { @@ -140,9 +141,11 @@ public ObjectSerializer( if (!isRecord && objectCreator.hasConstructorFields()) { constructorFieldIndexes = buildConstructorFieldIndexes(allFields); constructorFieldMask = buildConstructorFieldMask(allFields.length, constructorFieldIndexes); + constructorFieldTypes = constructorFieldTypes(); } else { constructorFieldIndexes = null; constructorFieldMask = null; + constructorFieldTypes = null; } } @@ -165,19 +168,6 @@ private void writeFields( } } - private void writeFields( - WriteContext writeContext, - T value, - RefWriter refWriter, - Generics generics, - boolean constructorFields) { - for (int i = 0; i < allFields.length; i++) { - if (constructorFieldMask[i] == constructorFields) { - writeFieldByCodecCategory(writeContext, value, refWriter, generics, allFields[i]); - } - } - } - private void printWriteFieldDebugInfo(SerializationFieldInfo fieldInfo, MemoryBuffer buffer) { LOG.info( "[Java] write field {} of type {}, writer index {}", @@ -308,8 +298,7 @@ private int countConstructorFields() { private T createConstructorObject(Object[] fieldValues) { return objectCreator.newInstanceWithArguments( - constructorArgs( - fieldValues, constructorFieldIndexes, objectCreator.getConstructorFieldTypes())); + constructorArgs(fieldValues, constructorFieldIndexes, constructorFieldTypes)); } private void setBufferedNonConstructorFields( @@ -338,19 +327,6 @@ public Object[] readFields(ReadContext readContext) { return fieldValues; } - private void readFields( - ReadContext readContext, Object[] fieldValues, boolean constructorFields) { - MemoryBuffer buffer = readContext.getBuffer(); - RefReader refReader = readContext.getRefReader(); - Generics generics = readContext.getGenerics(); - for (int i = 0; i < allFields.length; i++) { - if (constructorFieldMask[i] == constructorFields) { - fieldValues[i] = - readFieldByCodecCategory(readContext, refReader, generics, allFields[i], buffer); - } - } - } - public T readAndSetFields(ReadContext readContext, T obj) { MemoryBuffer buffer = readContext.getBuffer(); RefReader refReader = readContext.getRefReader(); @@ -365,17 +341,6 @@ public T readAndSetFields(ReadContext readContext, T obj) { return obj; } - private void readAndSetFields(ReadContext readContext, T obj, boolean constructorFields) { - MemoryBuffer buffer = readContext.getBuffer(); - RefReader refReader = readContext.getRefReader(); - Generics generics = readContext.getGenerics(); - for (int i = 0; i < allFields.length; i++) { - if (constructorFieldMask[i] == constructorFields) { - readAndSetFieldByCodecCategory(readContext, refReader, generics, allFields[i], buffer, obj); - } - } - } - private Object readFieldByCodecCategory( ReadContext readContext, RefReader refReader, diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectStreamSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectStreamSerializer.java index 6b14aad182..3a0e868e93 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectStreamSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectStreamSerializer.java @@ -692,8 +692,8 @@ private static class StreamTypeInfo { private final Consumer readObjectNoDataFunc; private StreamTypeInfo(Class type) { - // ReflectionFactory exposes Java serialization hooks without reading ObjectStreamClass - // private fields or requiring the java.io package to be opened. + // _JDKAccess owns version-specific discovery of serialization hooks without requiring + // ObjectStreamClass private-field access or a java.io package open. Method writeMethod = null; Method readMethod = null; Method noDataMethod = null; diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/StaticGeneratedStructSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/StaticGeneratedStructSerializer.java index 53b9a23df1..f348d64c2f 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/StaticGeneratedStructSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/StaticGeneratedStructSerializer.java @@ -249,12 +249,12 @@ public final int[] localFieldIds( } protected final int[] buildConstructorFieldIds(List descriptors) { - String[] fieldNames = objectCreator.getConstructorFieldNames(); + String[] fieldNames = constructorFieldNames(); if (fieldNames.length == 0) { return null; } - Class[] declaringClasses = objectCreator.getConstructorFieldDeclaringClasses(); - boolean[] finalFields = objectCreator.getConstructorFieldFinal(); + Class[] declaringClasses = constructorFieldDeclaringClasses(); + boolean[] finalFields = constructorFieldFinal(); int[] ids = new int[fieldNames.length]; for (int i = 0; i < fieldNames.length; i++) { Class declaringClass = declaringClasses == null ? null : declaringClasses[i]; diff --git a/java/fory-core/src/main/java/org/apache/fory/type/TypeUtils.java b/java/fory-core/src/main/java/org/apache/fory/type/TypeUtils.java index 8d0ded4193..58b01a8353 100644 --- a/java/fory-core/src/main/java/org/apache/fory/type/TypeUtils.java +++ b/java/fory-core/src/main/java/org/apache/fory/type/TypeUtils.java @@ -872,7 +872,9 @@ public static boolean isBean(TypeRef typeRef, TypeResolutionContext ctx) { || ctx.getCustomTypeRegistry().isExtraSupportedType(typeRef)) { return false; } - // if ReflectionUtils.hasNoArgConstructor(cls) return false, we use Unsafe to create object. + // Bean shape detection is independent of the later object-creation strategy. Construction may + // use a no-arg constructor, record/constructor-field metadata, or the platform empty-instance + // owner selected by TypeResolver. // bean class can be static nested class, but can't be not a non-static inner class if (cls.getEnclosingClass() != null && !Modifier.isStatic(cls.getModifiers())) { return false; diff --git a/java/fory-core/src/main/java16/module-info.java b/java/fory-core/src/main/java16/module-info.java index 7278f6cdf6..ce48341727 100644 --- a/java/fory-core/src/main/java16/module-info.java +++ b/java/fory-core/src/main/java16/module-info.java @@ -54,9 +54,4 @@ exports org.apache.fory.util; exports org.apache.fory.util.function; exports org.apache.fory.util.record; - // Sibling artifacts still use these internal JDK access helpers; keep this qualified - // export until those imports move behind public owner APIs. - exports org.apache.fory.platform.internal to - org.apache.fory.extension, - org.apache.fory.format; } diff --git a/java/fory-core/src/main/java25/module-info.java b/java/fory-core/src/main/java25/module-info.java index 25f4419f1f..3abd96aac0 100644 --- a/java/fory-core/src/main/java25/module-info.java +++ b/java/fory-core/src/main/java25/module-info.java @@ -53,10 +53,4 @@ exports org.apache.fory.util; exports org.apache.fory.util.function; exports org.apache.fory.util.record; - - // Sibling artifacts still use these internal JDK access helpers; keep this qualified - // export until those imports move behind public owner APIs. - exports org.apache.fory.platform.internal to - org.apache.fory.extension, - org.apache.fory.format; } diff --git a/java/fory-core/src/main/java25/org/apache/fory/reflect/FieldAccessorStrategy.java b/java/fory-core/src/main/java25/org/apache/fory/reflect/FieldAccessorStrategy.java index d56b42a0b3..afdedb918b 100644 --- a/java/fory-core/src/main/java25/org/apache/fory/reflect/FieldAccessorStrategy.java +++ b/java/fory-core/src/main/java25/org/apache/fory/reflect/FieldAccessorStrategy.java @@ -126,8 +126,7 @@ private static IllegalStateException finalMutationFailure(Field field, Throwable String versionMessage = JdkVersion.MAJOR_VERSION >= 26 ? "On JDK26+, start the JVM with " - + "--enable-final-field-mutation=org.apache.fory.core, or ALL-UNNAMED " - + "when Fory is loaded from the classpath. " + + "--enable-final-field-mutation=org.apache.fory.core. " : ""; return new IllegalStateException( "Cannot write final field " diff --git a/java/fory-core/src/main/java25/org/apache/fory/reflect/UnsafeObjectAllocator.java b/java/fory-core/src/main/java25/org/apache/fory/reflect/UnsafeObjectAllocator.java index 274e711ad6..35809db017 100644 --- a/java/fory-core/src/main/java25/org/apache/fory/reflect/UnsafeObjectAllocator.java +++ b/java/fory-core/src/main/java25/org/apache/fory/reflect/UnsafeObjectAllocator.java @@ -63,24 +63,34 @@ private static ForyException unsupported(Class type, Throwable cause) { } private static final class ObjectStreamClassAccess { - private static final MethodHandle NEW_INSTANCE = newInstanceHandle(); + private static final MethodHandle NEW_INSTANCE; + private static final Throwable INIT_ERROR; - private static MethodHandle newInstanceHandle() { + static { + MethodHandle newInstance = null; + Throwable error = null; try { - return _JDKAccess - ._trustedLookup(ObjectStreamClass.class) - .findVirtual( - ObjectStreamClass.class, "newInstance", MethodType.methodType(Object.class)); - } catch (ReflectiveOperationException e) { - throw new ForyException( - "JDK25+ Serializable object creation requires java.base/java.lang.invoke to be open " - + "to org.apache.fory.core", - e); + newInstance = + _JDKAccess + ._trustedLookup(ObjectStreamClass.class) + .findVirtual( + ObjectStreamClass.class, "newInstance", MethodType.methodType(Object.class)); + } catch (ReflectiveOperationException | RuntimeException e) { + error = e; } + NEW_INSTANCE = newInstance; + INIT_ERROR = error; } private static Object newInstance(Class type) throws Throwable { - return NEW_INSTANCE.invoke(ObjectStreamClass.lookupAny(type)); + MethodHandle handle = NEW_INSTANCE; + if (handle == null) { + throw new ForyException( + "JDK25+ Serializable object creation requires java.base/java.lang.invoke to be open " + + "to org.apache.fory.core", + INIT_ERROR); + } + return handle.invoke(ObjectStreamClass.lookupAny(type)); } } } diff --git a/java/fory-core/src/main/java9/module-info.java b/java/fory-core/src/main/java9/module-info.java index 22b688b441..49b67a908f 100644 --- a/java/fory-core/src/main/java9/module-info.java +++ b/java/fory-core/src/main/java9/module-info.java @@ -53,9 +53,4 @@ exports org.apache.fory.util; exports org.apache.fory.util.function; exports org.apache.fory.util.record; - // Sibling artifacts still use these internal JDK access helpers; keep this qualified - // export until those imports move behind public owner APIs. - exports org.apache.fory.platform.internal to - org.apache.fory.extension, - org.apache.fory.format; } diff --git a/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties b/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties index 57cd8cc996..e0549368c0 100644 --- a/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties +++ b/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties @@ -30,6 +30,7 @@ Args=--initialize-at-build-time=org.apache.fory.annotation.ForyField$Dynamic,\ org.apache.fory.memory.MemoryBuffer,\ org.apache.fory.memory.NativeByteOrder,\ org.apache.fory.platform.AndroidSupport,\ + org.apache.fory.reflect.ObjectCreatorRegistry,\ org.apache.fory.reflect.JvmTypeUseMetadata,\ org.apache.fory.reflect.TypeUseMetadata,\ org.apache.fory.resolver.StaticGeneratedSerializerRegistry,\ diff --git a/java/fory-core/src/test/java/org/apache/fory/TestUtils.java b/java/fory-core/src/test/java/org/apache/fory/TestUtils.java index 6fa564d930..746e640004 100644 --- a/java/fory-core/src/test/java/org/apache/fory/TestUtils.java +++ b/java/fory-core/src/test/java/org/apache/fory/TestUtils.java @@ -81,7 +81,7 @@ public static List javaCommand( private static List forkJvmArgs() { List args = new ArrayList<>(); if (JdkVersion.MAJOR_VERSION >= 25) { - args.add("--add-opens=java.base/java.lang.invoke=ALL-UNNAMED"); + args.add("--add-opens=java.base/java.lang.invoke=org.apache.fory.core"); if (hasInputArg("--sun-misc-unsafe-memory-access=deny")) { args.add("--sun-misc-unsafe-memory-access=deny"); } diff --git a/java/fory-core/src/test/java/org/apache/fory/serializer/AndroidJvmRoundTripTest.java b/java/fory-core/src/test/java/org/apache/fory/serializer/AndroidJvmRoundTripTest.java index e526deba4b..5b37f5a698 100644 --- a/java/fory-core/src/test/java/org/apache/fory/serializer/AndroidJvmRoundTripTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/serializer/AndroidJvmRoundTripTest.java @@ -159,7 +159,6 @@ private static void addAddOpens(ArrayList command) { return; } addAddOpens(command, "java.base/java.lang=ALL-UNNAMED"); - addAddOpens(command, "java.base/java.lang.invoke=ALL-UNNAMED"); addAddOpens(command, "java.base/java.lang.reflect=ALL-UNNAMED"); addAddOpens(command, "java.base/java.util=ALL-UNNAMED"); } diff --git a/java/fory-extensions/src/main/java/org/apache/fory/extension/serializer/ProtobufSerializer.java b/java/fory-extensions/src/main/java/org/apache/fory/extension/serializer/ProtobufSerializer.java index 9711e869a1..f10aa30bf1 100644 --- a/java/fory-extensions/src/main/java/org/apache/fory/extension/serializer/ProtobufSerializer.java +++ b/java/fory-extensions/src/main/java/org/apache/fory/extension/serializer/ProtobufSerializer.java @@ -32,7 +32,6 @@ import org.apache.fory.context.ReadContext; import org.apache.fory.context.WriteContext; import org.apache.fory.memory.MemoryBuffer; -import org.apache.fory.platform.internal._JDKAccess; import org.apache.fory.serializer.Serializer; import org.apache.fory.serializer.Shareable; import org.apache.fory.util.ExceptionUtils; @@ -46,7 +45,7 @@ public class ProtobufSerializer extends Serializer implements Shareable @Override protected MethodHandle[] computeValue(Class type) { try { - MethodHandles.Lookup lookup = _JDKAccess._trustedLookup(type); + MethodHandles.Lookup lookup = MethodHandles.publicLookup(); MethodHandle parseFrom1 = lookup.findStatic( type, "parseFrom", MethodType.methodType(type, CodedInputStream.class)); diff --git a/java/fory-format/pom.xml b/java/fory-format/pom.xml index c54ed44b2d..6cb04d742e 100644 --- a/java/fory-format/pom.xml +++ b/java/fory-format/pom.xml @@ -105,7 +105,6 @@ - true org.apache.fory.format true diff --git a/java/fory-latest-jdk-tests/src/test/java/org/apache/fory/integration_tests/ImmutableCollectionSerializersTest.java b/java/fory-latest-jdk-tests/src/test/java/org/apache/fory/integration_tests/ImmutableCollectionSerializersTest.java index 0b7983a120..ed1afbff76 100644 --- a/java/fory-latest-jdk-tests/src/test/java/org/apache/fory/integration_tests/ImmutableCollectionSerializersTest.java +++ b/java/fory-latest-jdk-tests/src/test/java/org/apache/fory/integration_tests/ImmutableCollectionSerializersTest.java @@ -21,13 +21,13 @@ import static org.apache.fory.integration_tests.TestUtils.serDeCheck; -import java.beans.ConstructorProperties; import java.util.List; import java.util.Map; import java.util.Set; import lombok.Data; import org.apache.fory.Fory; import org.apache.fory.ThreadSafeFory; +import org.apache.fory.annotation.ForyConstructor; import org.apache.fory.test.bean.CollectionFields; import org.apache.fory.test.bean.MapFields; import org.testng.Assert; @@ -97,7 +97,7 @@ public void testImmutableMapStruct() { public static class Pojo { List> data; - @ConstructorProperties("data") + @ForyConstructor("data") public Pojo(List> data) { this.data = data; } diff --git a/java/fory-test-core/src/main/java/org/apache/fory/test/bean/AccessBeans.java b/java/fory-test-core/src/main/java/org/apache/fory/test/bean/AccessBeans.java index 7b7dfab896..0ff485b42e 100644 --- a/java/fory-test-core/src/main/java/org/apache/fory/test/bean/AccessBeans.java +++ b/java/fory-test-core/src/main/java/org/apache/fory/test/bean/AccessBeans.java @@ -19,7 +19,6 @@ package org.apache.fory.test.bean; -import java.beans.ConstructorProperties; import lombok.Data; public class AccessBeans { @@ -29,7 +28,8 @@ private static class PrivateClass { int f2; private int f3; - @ConstructorProperties({"f1", "f2", "f3"}) + PrivateClass() {} + PrivateClass(int f1, int f2, int f3) { this.f1 = f1; this.f2 = f2; @@ -43,7 +43,8 @@ private static final class FinalPrivateClass { int f2; private int f3; - @ConstructorProperties({"f1", "f2", "f3"}) + FinalPrivateClass() {} + FinalPrivateClass(int f1, int f2, int f3) { this.f1 = f1; this.f2 = f2; @@ -57,7 +58,8 @@ static class DefaultLevelClass { int f2; private int f3; - @ConstructorProperties({"f1", "f2", "f3"}) + DefaultLevelClass() {} + DefaultLevelClass(int f1, int f2, int f3) { this.f1 = f1; this.f2 = f2; @@ -74,7 +76,8 @@ public static class PublicClass { private PrivateClass f5; private FinalPrivateClass f6; - @ConstructorProperties({"f1", "f2", "f3", "f4", "f5", "f6"}) + public PublicClass() {} + public PublicClass( int f1, int f2, int f3, DefaultLevelClass f4, PrivateClass f5, FinalPrivateClass f6) { this.f1 = f1; diff --git a/java/fory-testsuite/src/test/java/org/apache/fory/serializer/collection/GuavaCollectionSerializersTest.java b/java/fory-testsuite/src/test/java/org/apache/fory/serializer/collection/GuavaCollectionSerializersTest.java index 744ec9d0a7..f961e2d2e0 100644 --- a/java/fory-testsuite/src/test/java/org/apache/fory/serializer/collection/GuavaCollectionSerializersTest.java +++ b/java/fory-testsuite/src/test/java/org/apache/fory/serializer/collection/GuavaCollectionSerializersTest.java @@ -25,11 +25,11 @@ import com.google.common.collect.ImmutableSet; import com.google.common.collect.ImmutableSortedMap; import com.google.common.collect.ImmutableSortedSet; -import java.beans.ConstructorProperties; import java.util.List; import java.util.Objects; import org.apache.fory.Fory; import org.apache.fory.TestBase; +import org.apache.fory.annotation.ForyConstructor; import org.testng.Assert; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; @@ -230,7 +230,7 @@ public void testNestedRefTrackingCopy(Fory fory) { public static final class Pojo { private final List> data; - @ConstructorProperties("data") + @ForyConstructor("data") public Pojo(List> data) { this.data = data; } diff --git a/java/fory-testsuite/src/test/java/org/apache/fory/test/FastJsonTest.java b/java/fory-testsuite/src/test/java/org/apache/fory/test/FastJsonTest.java index 08cdecf248..2c89a3bf3a 100644 --- a/java/fory-testsuite/src/test/java/org/apache/fory/test/FastJsonTest.java +++ b/java/fory-testsuite/src/test/java/org/apache/fory/test/FastJsonTest.java @@ -22,9 +22,9 @@ import com.alibaba.fastjson.JSONObject; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Sets; -import java.beans.ConstructorProperties; import java.util.List; import org.apache.fory.Fory; +import org.apache.fory.annotation.ForyConstructor; import org.apache.fory.collection.Collections; import org.testng.Assert; import org.testng.annotations.DataProvider; @@ -35,7 +35,7 @@ public static class DemoResponse { private JSONObject json; private List objects; - @ConstructorProperties("json") + @ForyConstructor("json") public DemoResponse(JSONObject json) { this.json = json; objects = Collections.ofArrayList(json); diff --git a/kotlin/fory-kotlin-ksp/src/main/kotlin/org/apache/fory/kotlin/ksp/KotlinSerializerSourceWriter.kt b/kotlin/fory-kotlin-ksp/src/main/kotlin/org/apache/fory/kotlin/ksp/KotlinSerializerSourceWriter.kt index 6c91ae89c0..afcee53408 100644 --- a/kotlin/fory-kotlin-ksp/src/main/kotlin/org/apache/fory/kotlin/ksp/KotlinSerializerSourceWriter.kt +++ b/kotlin/fory-kotlin-ksp/src/main/kotlin/org/apache/fory/kotlin/ksp/KotlinSerializerSourceWriter.kt @@ -94,6 +94,7 @@ internal class KotlinSerializerSourceWriter(private val struct: KotlinSourceStru builder.append(" private val fieldsById: Array\n") builder.append(" private val constructorFieldIds: IntArray?\n") builder.append(" private val constructorFieldBits: LongArray?\n") + builder.append(" private val constructorFieldTypes: Array>?\n") builder.append(" private val classVersionHash: Int\n") builder.append(" private val sameSchemaCompatible: Boolean\n\n") } @@ -138,6 +139,7 @@ internal class KotlinSerializerSourceWriter(private val struct: KotlinSourceStru builder.append(" this.fieldsById = arrayOfNulls(0)\n") builder.append(" this.constructorFieldIds = null\n") builder.append(" this.constructorFieldBits = null\n") + builder.append(" this.constructorFieldTypes = null\n") builder.append(" this.classVersionHash = 0\n") builder.append(" this.sameSchemaCompatible = false\n") builder.append(" }\n\n") @@ -172,6 +174,9 @@ internal class KotlinSerializerSourceWriter(private val struct: KotlinSourceStru builder.append( " this.constructorFieldBits = buildConstructorFieldBits(DESCRIPTORS.size, constructorFieldIds)\n" ) + builder.append( + " this.constructorFieldTypes = if (constructorFieldIds != null) constructorFieldTypes() else null\n" + ) writeScalarBindings() builder.append( " this.classVersionHash = if (typeResolver.checkClassVersion()) computeClassVersionHash(DESCRIPTORS) else 0\n" @@ -348,7 +353,7 @@ internal class KotlinSerializerSourceWriter(private val struct: KotlinSourceStru .append(struct.typeName) .append(" {\n") builder.append( - " return objectCreator.newInstanceWithArguments(*constructorArgs(fieldValues, constructorFieldIds!!, objectCreator.getConstructorFieldTypes())) as " + " return objectCreator.newInstanceWithArguments(*constructorArgs(fieldValues, constructorFieldIds!!, constructorFieldTypes!!)) as " ) builder.append(struct.typeName).append("\n") builder.append(" }\n\n") diff --git a/scala/build.sbt b/scala/build.sbt index 1015c20fa4..f72df29bee 100644 --- a/scala/build.sbt +++ b/scala/build.sbt @@ -48,9 +48,6 @@ libraryDependencies ++= Seq( ) Test / fork := true -Test / javaOptions ++= Seq( - "--add-opens=java.base/java.lang.invoke=ALL-UNNAMED", -) lazy val writeTestClasspath = taskKey[File]("Writes the Scala test runtime classpath") diff --git a/scala/src/main/scala/org/apache/fory/serializer/scala/RangeSerializer.scala b/scala/src/main/scala/org/apache/fory/serializer/scala/RangeSerializer.scala index 3fa114f104..8bfb30e4e2 100644 --- a/scala/src/main/scala/org/apache/fory/serializer/scala/RangeSerializer.scala +++ b/scala/src/main/scala/org/apache/fory/serializer/scala/RangeSerializer.scala @@ -27,7 +27,6 @@ import org.apache.fory.serializer.Serializer import org.apache.fory.serializer.collection.CollectionLikeSerializer import org.apache.fory.resolver.TypeResolver import java.util -import org.apache.fory.platform.internal._JDKAccess import java.lang.invoke.{MethodHandle, MethodHandles} import scala.collection.immutable.NumericRange @@ -63,10 +62,10 @@ class RangeSerializer[T <: Range](typeResolver: TypeResolver, cls: Class[T]) private object RangeUtils { - val lookupCache: ClassValue[MethodHandle] = new ClassValue[MethodHandle]() { + private val publicLookup = MethodHandles.publicLookup() + val lookupCache: ClassValue[MethodHandle] = new ClassValue[MethodHandle]() { override protected def computeValue(cls: Class[_]): MethodHandle = { - val lookup: MethodHandles.Lookup = _JDKAccess._trustedLookup(cls) - lookup.unreflectConstructor(cls.getDeclaredConstructors()(0)) + publicLookup.unreflectConstructor(cls.getConstructors()(0)) } } } From 5692ad8adde64f16504da8056afa9c855bb47c96 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Tue, 2 Jun 2026 09:26:31 +0800 Subject: [PATCH 38/69] fix errors --- .agents/languages/java.md | 35 +- .agents/languages/kotlin.md | 7 + ci/run_ci.sh | 4 +- ci/tasks/java.py | 7 +- docs/guide/java/native-serialization.md | 14 +- docs/guide/kotlin/configuration.md | 21 +- docs/guide/kotlin/schema-metadata.md | 13 +- .../kotlin/static-generated-serializers.md | 15 +- .../jdk_compatibility_tests/pom.xml | 5 - .../fory/integration_tests/ForyTest.java | 75 +++- .../JDKCompatibilityTest.java | 31 +- integration_tests/jpms_tests/pom.xml | 5 - .../jpms_tests/src/main/java/module-info.java | 3 +- .../apache/fory/integration_tests/Test.java | 3 +- .../StaticSerializerSourceWriter.java | 3 + .../processing/ForyStructProcessorTest.java | 38 +++ .../apache/fory/AbstractThreadSafeFory.java | 23 +- .../main/java/org/apache/fory/BaseFory.java | 9 +- .../src/main/java/org/apache/fory/Fory.java | 8 + .../java/org/apache/fory/ThreadLocalFory.java | 15 + .../fory/annotation/ForyConstructor.java | 3 + .../fory/builder/UnsafeCodegenSupport.java | 4 +- .../org/apache/fory/codegen/Expression.java | 4 +- .../org/apache/fory/memory/LittleEndian.java | 7 +- .../org/apache/fory/pool/ThreadPoolFory.java | 15 + .../fory/reflect/ObjectCreatorRegistry.java | 3 +- .../apache/fory/reflect/ObjectCreators.java | 111 ++++-- .../apache/fory/resolver/ClassResolver.java | 13 - .../serializer/AbstractObjectSerializer.java | 154 ++++++++- .../CompatibleCollectionArrayReader.java | 4 +- .../fory/serializer/ExceptionSerializers.java | 29 +- .../serializer/ObjectStreamSerializer.java | 20 +- .../fory/serializer/PlatformStringUtils.java | 6 +- .../apache/fory/serializer/Serializers.java | 115 ++----- .../StaticGeneratedStructSerializer.java | 3 - .../collection/CollectionSerializers.java | 29 +- .../fory/builder/UnsafeCodegenSupport.java | 2 +- .../org/apache/fory/memory/MemoryBuffer.java | 14 +- .../fory-core/native-image.properties | 3 +- .../test/java/org/apache/fory/ForyTest.java | 13 + .../fory/JpmsOptionalClassLoadingTest.java | 4 + .../org/apache/fory/ThreadSafeForyTest.java | 120 ++++++- .../apache/fory/memory/MemoryBufferTest.java | 4 +- .../fory/reflect/ObjectCreatorsTest.java | 14 + .../fory/serializer/ObjectSerializerTest.java | 319 +++++++++++++++++- .../ObjectStreamSerializerTest.java | 217 +++++++++++- .../fory/serializer/URLSerializerTest.java | 2 +- .../otherpkg/PackageNoArgParent.java | 26 ++ .../ImmutableCollectionSerializersTest.java | 23 ++ java/pom.xml | 1 - .../kotlin/ksp/ForyKotlinSymbolProcessor.kt | 19 +- .../ksp/KotlinSerializerSourceWriter.kt | 147 +++++--- .../kotlin/ksp/ProcessorValidationTest.kt | 62 ++++ kotlin/fory-kotlin-tests/pom.xml | 1 - .../fory/kotlin/xlang/KotlinXlangPeer.kt | 181 +++++++++- kotlin/fory-kotlin/pom.xml | 2 - .../serializer/kotlin/KotlinSerializers.java | 18 - .../kotlin/KotlinBuiltinSerializers.kt | 236 ------------- 58 files changed, 1690 insertions(+), 592 deletions(-) create mode 100644 java/fory-core/src/test/java/org/apache/fory/serializer/otherpkg/PackageNoArgParent.java delete mode 100644 kotlin/fory-kotlin/src/main/kotlin/org/apache/fory/serializer/kotlin/KotlinBuiltinSerializers.kt diff --git a/.agents/languages/java.md b/.agents/languages/java.md index 6be6bb2374..9c8e2f30ce 100644 --- a/.agents/languages/java.md +++ b/.agents/languages/java.md @@ -46,6 +46,9 @@ Load this file when changing anything under `java/` or when Java drives a cross- `MemoryOps` call, keep JVM bulk loads/stores local with raw Unsafe operations instead of routing through branchful `_unsafeGet*` or `_unsafePut*` helpers. Add or preserve source comments that explain this inlining invariant when editing these methods. +- Unsafe object-offset arithmetic must stay `long` before calls such as `getLong(Object, long)` or + `putLong(Object, long)`. An all-`int` expression can compile under JDK8 to a bytecode descriptor + that fails with `NoSuchMethodError` on JDK9+. - In primitive-array swap-endian readers, do not loop through `MemoryBuffer._unsafeGet*` helpers. Copy the payload through the typed payload API, then swap destination values locally so the path stays stream-safe and avoids Android-dispatch helper drift. @@ -56,7 +59,37 @@ Load this file when changing anything under `java/` or when Java drives a cross- - Keep JDK25+ unsafe-removal implementation invariants in agent/design docs and tests, not user guides. User guides should document user actions such as `--sun-misc-unsafe-memory-access=deny`, `java.base/java.lang.invoke` opens, JDK26+ `--enable-final-field-mutation`, package opens for user named modules, and public APIs such as `@ForyConstructor` and `BaseFory.registerConstructor(...)`; do not expose internal serializer names, owner-model rationale, or avoided fallback strategies there. - For JDK25+ zero-Unsafe final-field behavior, distinguish JDK25 from JDK26+: JDK25 has no final-field mutation flag requirement, while JDK26+ requires `--enable-final-field-mutation` for post-construction final-field writes. - For JDK25+ object creation, do not use `sun.reflect.ReflectionFactory` or `jdk.internal.reflect.ReflectionFactory`. Serializable classes without a no-arg constructor may use `ObjectStreamClass.newInstance` through the trusted lookup owner; non-Serializable classes without a no-arg constructor require `@ForyConstructor`, `BaseFory.registerConstructor(...)`, record construction, or a custom serializer. +- Treat `ByteArrayOutputStream` and `ByteArrayInputStream` as ordinary streams on every JDK. Do + not restore private-buffer wrapping for JDK8-24 performance, because that reintroduces + `java.base/java.io` private-field ownership and module-open requirements. +- `ObjectStreamSerializer` must use its stream-specific object creator path and must not use + `TypeResolver.getObjectCreator`. `@ForyConstructor` and registered constructor mappings require + field values up front, while ObjectStream reconstruction creates the object before stream fields + are read. This path must not invoke Serializable class constructors, including no-arg + constructors; the JDK8-24 ReflectionFactory template constructor must be the first + non-Serializable superclass no-arg constructor, and private no-arg constructors must be rejected + instead of made accessible. Package-private no-arg constructors are valid only for same-package + Serializable subclasses. +- Root codegen and builder classes that still need Unsafe on JDK8-24 must route symbolic Unsafe + access through a helper with a Java 25 replacement. Do not leave `_JDKAccess.unsafe()` or + `sun.misc.Unsafe` references in JDK25-visible classes outside matching `java25` replacements. +- JDK25+ collection serializers must fail unsupported `Collections.newSetFromMap` backing maps + before writing or copying. Do not rewrite them to `HashMap`, because that changes equality + semantics and can drop entries. +- Do not enable Java or Kotlin parameter-name metadata (`-parameters`, `maven.compiler.parameters`, or Kotlin `javaParameters`) to make constructor binding work. Constructor binding must be driven by explicit field mappings from `@ForyConstructor`, `BaseFory.registerConstructor(...)`, or generated metadata. - Constructor-bound serializers must cache constructor field metadata during serializer initialization. Do not call defensive-copy metadata getters such as `getConstructorFieldTypes()` from per-object read paths. +- Explicit constructor binding is for user and third-party classes, not Java platform types. Do not use + `java.*` internals such as `String` fields as constructor-binding test data; JDK built-in classes + are owned by their serializers. +- Runtime serializers and generated serializers must use the same constructor-copy lifecycle: + install a pending marker before copying constructor-bound fields, run field serializers normally, + reject copied constructor arguments that still retain the marker, then construct and reference the + real copy. Do not implement a separate raw-field pre-scan or leave a generated path without the + marker guard. +- When a `Throwable` is created without running `Throwable` constructors, restore the private + `cause` and `suppressedExceptions` sentinels directly before exposing the object. Do not call + `initCause` or `addSuppressed` on a constructor-bypassed `Throwable` whose sentinels are still + absent. - For JDK25+ CI, do not run core runtime tests from raw Maven reactor `target/classes`. Those classes bypass `META-INF/versions/25` and exercise the JDK8-24 root implementation. Build/install the multi-release artifact first, then verify the zero-Unsafe path through the JPMS module-path suite where `org.apache.fory.core` is the real access target. - Do not make GraalVM native-image JDK25+ pass by opening `java.lang.invoke` to `ALL-UNNAMED`. Keep zero-Unsafe verification on JPMS JVM tests unless the native-image path itself runs Fory as a named module and the produced binary passes. @@ -108,4 +141,4 @@ mvn -T16 test -Dtest=org.apache.fory.TestClass#testMethod - Set `ENABLE_FORY_GENERATED_CLASS_UNIQUE_ID=false` when you need stable generated class names. - When debugging Java tests or runtime behavior, set `FORY_LOG_LEVEL=INFO` unless a narrower level is required. -- In IntelliJ IDEA, use a JDK 11+ project SDK and disable `--release` if it blocks `sun.misc.Unsafe` access. +- In IntelliJ IDEA, use the same JDK and module flags as the Maven profile you are debugging. diff --git a/.agents/languages/kotlin.md b/.agents/languages/kotlin.md index 2855fc6367..7e45aab2f7 100644 --- a/.agents/languages/kotlin.md +++ b/.agents/languages/kotlin.md @@ -6,6 +6,13 @@ Load this file when changing `kotlin/`. - Run Kotlin Maven commands from within `kotlin/`. - Kotlin serializers build on the Java implementation. If Java changed and the updated Java artifacts are not installed yet, run `cd ../java && mvn -T16 install -DskipTests` first. +- KSP `@ForyStruct` serializers that use a primary constructor must require explicit + `@ForyConstructor` field mappings. Do not bind constructor fields from Kotlin parameter names or + `javaParameters`; mutable no-argument structs should use `var` properties with `@ForyField`. +- Preserve serializer-family selection for Kotlin standard-library types already registered by + Fory. Do not auto-install a new serializer for an existing type-registered Kotlin class unless the + wire format matches the previous serializer family and old-payload/new-runtime compatibility is + tested. ## Commands diff --git a/ci/run_ci.sh b/ci/run_ci.sh index ca52828ffd..80db64756f 100755 --- a/ci/run_ci.sh +++ b/ci/run_ci.sh @@ -163,7 +163,7 @@ install_jdk25_fory_artifacts() { use_jdk "zulu25.30.17-ca-jdk25.0.1-linux_x64" export JDK_JAVA_OPTIONS="$(jdk25_javac_options)" cd "$ROOT"/java - mvn -T10 -B --no-transfer-progress clean install -DskipTests -Dmaven.compiler.parameters=true -pl '!:fory-testsuite' + mvn -T10 -B --no-transfer-progress clean install -DskipTests -pl '!:fory-testsuite' echo "Verify JDK25 benchmark multi-release jar" cd "$ROOT"/benchmarks/java mvn -T10 -B --no-transfer-progress -Pjmh -DskipTests install @@ -237,7 +237,7 @@ jdk17_plus_tests() { # JDK25+ must be tested from the packaged multi-release artifact. Raw # reactor test classes bypass META-INF/versions/25 and exercise the # JDK8-24 root implementation instead. - mvn -T10 --batch-mode --no-transfer-progress clean install -DskipTests -Dmaven.compiler.parameters=true + mvn -T10 --batch-mode --no-transfer-progress clean install -DskipTests else mvn -T10 --batch-mode --no-transfer-progress install fi diff --git a/ci/tasks/java.py b/ci/tasks/java.py index b35946bfac..022abc4241 100644 --- a/ci/tasks/java.py +++ b/ci/tasks/java.py @@ -56,7 +56,7 @@ def get_jdk_major_version(): "21": "zulu21.28.85-ca-jdk21.0.0-linux_x64", "24": "zulu24.32.13-ca-fx-jdk24.0.2-linux_x64", "25": "zulu25.30.17-ca-jdk25.0.1-linux_x64", - "26": "zulu26.28.89-ca-jdk26.0.0-linux_x64", + "26": "zulu26.30.11-ca-crac-jdk26.0.1-linux_x64", } @@ -151,7 +151,7 @@ def install_jdk25_fory_artifacts(): common.cd_project_subdir("java") common.exec_cmd( "mvn -T10 -B --no-transfer-progress clean install -DskipTests " - "-Dmaven.compiler.parameters=true -pl '!:fory-testsuite'" + "-pl '!:fory-testsuite'" ) logging.info("Verify JDK25 benchmark multi-release jar") common.cd_project_subdir("benchmarks/java") @@ -268,8 +268,7 @@ def run_jdk17_plus(java_version="17"): # reactor test classes bypass META-INF/versions/25 and exercise the # JDK8-24 root implementation instead. common.exec_cmd( - "mvn -T10 --batch-mode --no-transfer-progress clean install " - "-DskipTests -Dmaven.compiler.parameters=true" + "mvn -T10 --batch-mode --no-transfer-progress clean install -DskipTests" ) os.environ.pop("JDK_JAVA_OPTIONS", None) logging.info(f"Executing JDK{java_version} JPMS tests") diff --git a/docs/guide/java/native-serialization.md b/docs/guide/java/native-serialization.md index 58e1392dff..b3596ce7ee 100644 --- a/docs/guide/java/native-serialization.md +++ b/docs/guide/java/native-serialization.md @@ -184,7 +184,8 @@ public final class User { } ``` -For third-party classes that cannot be annotated, register the constructor during runtime setup: +For third-party classes that cannot be annotated, register the constructor during runtime setup +before requesting serializers or starting serialization, deserialization, or copy operations: ```java import java.lang.reflect.Constructor; @@ -195,6 +196,9 @@ fory.registerConstructor(User.class, constructor, "name", "age"); The field names are the binding contract. For ordinary classes, Fory does not infer constructor bindings from Java parameter names, `-parameters`, or `@ConstructorProperties`. +Explicit constructor mappings must bind at least one field. Leave ordinary no-argument constructors +unannotated and unregistered. Do not register constructors for Java platform classes; Fory owns +their built-in serializers. When no explicit constructor mapping is provided, normal classes with final fields use Fory's normal object creation and field setting. On JDK25+ with Unsafe memory access denied, Fory reports an error @@ -206,9 +210,11 @@ mutation flag. See [Troubleshooting](troubleshooting.md#jdk25-zero-unsafe-mode-and-module-opens) for the required JVM flags. -Constructor-bound objects cannot receive a constructor argument that refers directly to the same -object under construction. Model those cycles through non-constructor fields or use a custom -serializer. +Constructor-bound objects cannot receive a constructor argument that retains a direct or nested +back-reference to the same object under construction. Field serializers may transform or omit data +during copy; Fory rejects only back-references that remain in the copied constructor arguments. Model +those cycles through non-constructor fields or use a custom serializer that removes the constructor +argument cycle. ## JDK Serialization Hooks diff --git a/docs/guide/kotlin/configuration.md b/docs/guide/kotlin/configuration.md index a2fcaebce3..eb37157c2c 100644 --- a/docs/guide/kotlin/configuration.md +++ b/docs/guide/kotlin/configuration.md @@ -103,6 +103,8 @@ On JDK25+ with Unsafe memory access denied, Kotlin classes with final constructo an explicit constructor mapping when Fory must call the primary constructor. Annotate the constructor with `@ForyConstructor`, register the constructor with `registerConstructor(...)`, use a generated serializer that carries explicit constructor metadata, or provide a custom serializer. +Register constructors during setup before requesting serializers or starting serialization, +deserialization, or copy operations. ```kotlin import org.apache.fory.annotation.ForyConstructor @@ -113,22 +115,9 @@ class User @ForyConstructor("name", "age") constructor( ) ``` -For generated serializers and tooling that read Java parameter metadata, enable Java parameter -metadata for Kotlin compilation: - -```kotlin -kotlin { - compilerOptions { - javaParameters = true - } -} -``` - -For Maven builds, configure the Kotlin Maven plugin with: - -```xml -true -``` +KSP-generated `@ForyStruct` serializers that call a primary constructor require the same explicit +`@ForyConstructor` mapping. Mutable no-argument `@ForyStruct` classes can instead expose serialized +`var` properties with `@ForyField`. The JVM also needs the module opens and final-field mutation option listed in [Java Troubleshooting](../java/troubleshooting.md#jdk25-zero-unsafe-mode-and-module-opens). diff --git a/docs/guide/kotlin/schema-metadata.md b/docs/guide/kotlin/schema-metadata.md index 7b182ad01b..b27720d9b2 100644 --- a/docs/guide/kotlin/schema-metadata.md +++ b/docs/guide/kotlin/schema-metadata.md @@ -29,13 +29,16 @@ Annotate Kotlin schema classes with `@ForyStruct` and constructor properties wit `@ForyField(id = N)`: ```kotlin +import org.apache.fory.annotation.ForyConstructor import org.apache.fory.annotation.ForyField import org.apache.fory.annotation.ForyStruct import org.apache.fory.kotlin.Fixed import org.apache.fory.kotlin.VarInt @ForyStruct -data class User( +data class User +@ForyConstructor("id", "score", "tags") +constructor( @ForyField(id = 1) val id: @Fixed UInt, @@ -58,7 +61,9 @@ and maps: ```kotlin @ForyStruct -data class NullabilityExample( +data class NullabilityExample +@ForyConstructor("names", "optionalNames", "nullableList") +constructor( @ForyField(id = 1) val names: List, @@ -81,7 +86,9 @@ Kotlin generated serializers preserve `@Ref` metadata for fields, list elements, import org.apache.fory.annotation.Ref @ForyStruct -data class Node( +data class Node +@ForyConstructor("children", "parent") +constructor( @ForyField(id = 1) val children: List<@Ref Node>, diff --git a/docs/guide/kotlin/static-generated-serializers.md b/docs/guide/kotlin/static-generated-serializers.md index 97f8150e1e..3bf3d2c713 100644 --- a/docs/guide/kotlin/static-generated-serializers.md +++ b/docs/guide/kotlin/static-generated-serializers.md @@ -54,13 +54,16 @@ Reuse the Java Fory annotations for schema concepts. Use Kotlin type-use annotations only when you need to override integer encoding. ```kotlin +import org.apache.fory.annotation.ForyConstructor import org.apache.fory.annotation.ForyField import org.apache.fory.annotation.ForyStruct import org.apache.fory.kotlin.Fixed import org.apache.fory.kotlin.VarInt @ForyStruct -data class User( +data class User +@ForyConstructor("id", "score", "tags") +constructor( @ForyField(id = 1) val id: @Fixed UInt, @@ -81,8 +84,10 @@ them. The processor generates serializers for public or internal, concrete, non-generic classes in named packages. A supported class must have a primary -constructor whose serialized parameters are `val` or `var` properties. `data -class` is the common case, but it is not required. +constructor whose serialized parameters are `val` or `var` properties and whose +constructor parameters are explicitly mapped with `@ForyConstructor`. `data +class` is the common case, but it is not required. Mutable no-argument structs +can instead expose serialized `var` properties with `@ForyField`. Internal Kotlin struct classes are supported when KSP runs in the same Kotlin module that owns the struct. The generated Kotlin serializer is also internal, @@ -115,7 +120,9 @@ inside collections and maps. ```kotlin @ForyStruct -data class NullabilityExample( +data class NullabilityExample +@ForyConstructor("a", "b", "c", "d") +constructor( @ForyField(id = 1) val a: List, diff --git a/integration_tests/jdk_compatibility_tests/pom.xml b/integration_tests/jdk_compatibility_tests/pom.xml index 08a1257b24..9df4aeb431 100644 --- a/integration_tests/jdk_compatibility_tests/pom.xml +++ b/integration_tests/jdk_compatibility_tests/pom.xml @@ -48,11 +48,6 @@ fory-format ${project.version} - - org.apache.fory - benchmark - ${project.version} - org.testng testng diff --git a/integration_tests/jdk_compatibility_tests/src/test/java/org/apache/fory/integration_tests/ForyTest.java b/integration_tests/jdk_compatibility_tests/src/test/java/org/apache/fory/integration_tests/ForyTest.java index 527988d2bc..6349a85f7d 100644 --- a/integration_tests/jdk_compatibility_tests/src/test/java/org/apache/fory/integration_tests/ForyTest.java +++ b/integration_tests/jdk_compatibility_tests/src/test/java/org/apache/fory/integration_tests/ForyTest.java @@ -20,9 +20,10 @@ package org.apache.fory.integration_tests; import java.io.Serializable; +import java.util.Arrays; +import java.util.List; +import java.util.Objects; import org.apache.fory.Fory; -import org.apache.fory.benchmark.data.MediaContent; -import org.apache.fory.benchmark.data.Sample; import org.apache.fory.platform.JdkVersion; import org.testng.Assert; import org.testng.annotations.Test; @@ -30,21 +31,13 @@ public class ForyTest { @Test - public void testMediaContent() { - Sample object = new Sample().populate(false); + public void testClasspathBeanGraph() { + ClasspathBean object = + new ClasspathBean( + "graph", Arrays.asList(new ClasspathItem("left", 1), new ClasspathItem("right", 2))); Fory fory = Fory.builder().withXlang(false).requireClassRegistration(false).build(); byte[] data = fory.serialize(object); - Sample sample = (Sample) fory.deserialize(data); - Assert.assertEquals(sample, object); - } - - @Test - public void testSample() { - MediaContent object = new MediaContent().populate(false); - Fory fory = Fory.builder().withXlang(false).requireClassRegistration(false).build(); - byte[] data = fory.serialize(object); - MediaContent mediaContent = (MediaContent) fory.deserialize(data); - Assert.assertEquals(mediaContent, object); + Assert.assertEquals(fory.deserialize(data), object); } @Test @@ -89,4 +82,56 @@ public int hashCode() { return name.hashCode() * 31 + age; } } + + static final class ClasspathBean implements Serializable { + private static final long serialVersionUID = 1L; + + private String name; + private List items; + + private ClasspathBean(String name, List items) { + this.name = name; + this.items = items; + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof ClasspathBean)) { + return false; + } + ClasspathBean other = (ClasspathBean) obj; + return Objects.equals(name, other.name) && Objects.equals(items, other.items); + } + + @Override + public int hashCode() { + return Objects.hash(name, items); + } + } + + static final class ClasspathItem implements Serializable { + private static final long serialVersionUID = 1L; + + private String name; + private int count; + + private ClasspathItem(String name, int count) { + this.name = name; + this.count = count; + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof ClasspathItem)) { + return false; + } + ClasspathItem other = (ClasspathItem) obj; + return count == other.count && Objects.equals(name, other.name); + } + + @Override + public int hashCode() { + return Objects.hash(name, count); + } + } } diff --git a/integration_tests/jdk_compatibility_tests/src/test/java/org/apache/fory/integration_tests/JDKCompatibilityTest.java b/integration_tests/jdk_compatibility_tests/src/test/java/org/apache/fory/integration_tests/JDKCompatibilityTest.java index 24a1ad9dd8..a24b1d8a0c 100644 --- a/integration_tests/jdk_compatibility_tests/src/test/java/org/apache/fory/integration_tests/JDKCompatibilityTest.java +++ b/integration_tests/jdk_compatibility_tests/src/test/java/org/apache/fory/integration_tests/JDKCompatibilityTest.java @@ -85,12 +85,10 @@ public void testSchemaConsist() throws IOException { Fory fory = builder().build(); fory.register(CustomObject.class); File dir = new File("."); - File[] files = dir.listFiles((d, name) -> name.startsWith("object_schema_consistent")); - assert files != null; + File[] files = preJdk25Files(dir, "object_schema_consistent"); check(object, fory, files); CustomObject customObject = createCustomObject(); - File[] files1 = dir.listFiles((d, name) -> name.startsWith("custom_object_schema_consistent")); - assert files1 != null; + File[] files1 = preJdk25Files(dir, "custom_object_schema_consistent"); check(customObject, fory, files1); } @@ -100,15 +98,32 @@ public void testSchemaCompatible() throws IOException { Fory fory = builder().withCompatible(true).build(); fory.register(CustomObject.class); File dir = new File("."); - File[] files = dir.listFiles((d, name) -> name.startsWith("object_schema_compatible")); - assert files != null; + File[] files = preJdk25Files(dir, "object_schema_compatible"); check(object, fory, files); CustomObject customObject = createCustomObject(); - File[] files1 = dir.listFiles((d, name) -> name.startsWith("custom_object_schema_compatible")); - assert files1 != null; + File[] files1 = preJdk25Files(dir, "custom_object_schema_compatible"); check(customObject, fory, files1); } + private static File[] preJdk25Files(File dir, String prefix) { + File[] files = + dir.listFiles((d, name) -> name.startsWith(prefix) && isPreJdk25Fixture(name, prefix)); + assert files != null; + return files; + } + + private static boolean isPreJdk25Fixture(String name, String prefix) { + String suffix = name.substring(prefix.length()); + if (suffix.isEmpty()) { + return false; + } + try { + return Integer.parseInt(suffix) < 25; + } catch (NumberFormatException e) { + return false; + } + } + private static void check(Object object, Fory fory, File[] files) throws IOException { for (File file : files) { byte[] bytes = Files.readAllBytes(file.toPath()); diff --git a/integration_tests/jpms_tests/pom.xml b/integration_tests/jpms_tests/pom.xml index 07ec323c74..15a92a1e86 100644 --- a/integration_tests/jpms_tests/pom.xml +++ b/integration_tests/jpms_tests/pom.xml @@ -40,11 +40,6 @@ - - org.apache.fory - benchmark - ${project.version} - org.apache.fory fory-core diff --git a/integration_tests/jpms_tests/src/main/java/module-info.java b/integration_tests/jpms_tests/src/main/java/module-info.java index 02eb446d24..c7e714742d 100644 --- a/integration_tests/jpms_tests/src/main/java/module-info.java +++ b/integration_tests/jpms_tests/src/main/java/module-info.java @@ -19,7 +19,6 @@ module org.apache.fory.integration_tests { - requires org.apache.fory.benchmark; requires org.apache.fory.core; requires org.apache.fory.format; requires org.apache.fory.test.core; @@ -29,5 +28,5 @@ exports org.apache.fory.integration_tests.model; exports org.apache.fory.integration_tests.publicserializer; - opens org.apache.fory.integration_tests.model to org.apache.fory.core, org.apache.fory.format; + opens org.apache.fory.integration_tests.model to org.apache.fory.core; } diff --git a/integration_tests/jpms_tests/src/main/java/org/apache/fory/integration_tests/Test.java b/integration_tests/jpms_tests/src/main/java/org/apache/fory/integration_tests/Test.java index 56ac832e75..2e3d778034 100644 --- a/integration_tests/jpms_tests/src/main/java/org/apache/fory/integration_tests/Test.java +++ b/integration_tests/jpms_tests/src/main/java/org/apache/fory/integration_tests/Test.java @@ -20,7 +20,6 @@ package org.apache.fory.integration_tests; import org.apache.fory.Fory; -import org.apache.fory.benchmark.Benchmark; import org.apache.fory.format.encoder.Encoders; import org.apache.fory.test.bean.Foo; @@ -34,6 +33,6 @@ public static void main(String[] args) { final Fory fory = Fory.builder().build(); fory.serialize(Foo.create()); - Encoders.bean(Benchmark.class, fory); + Encoders.bean(Foo.class, fory); } } diff --git a/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/StaticSerializerSourceWriter.java b/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/StaticSerializerSourceWriter.java index 596b43113a..4fdabc911c 100644 --- a/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/StaticSerializerSourceWriter.java +++ b/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/StaticSerializerSourceWriter.java @@ -1308,6 +1308,7 @@ private void writeConstructorCopy() { .append(struct.typeName) .append(" value) {\n"); builder.append(" Object[] fieldValues = new Object[DESCRIPTORS.size()];\n"); + builder.append(" Object pendingMarker = beginConstructorCopy(copyContext, value);\n"); for (SourceField field : struct.fields) { builder.append(" if (hasField(constructorFieldBits, ").append(field.id).append(")) {\n"); builder @@ -1320,6 +1321,8 @@ private void writeConstructorCopy() { .append("]);\n"); builder.append(" }\n"); } + builder.append( + " checkNoConstructorCopyBackrefs(fieldValues, constructorFieldIds, pendingMarker);\n"); builder .append(" ") .append(struct.typeName) diff --git a/java/fory-annotation-processor/src/test/java/org/apache/fory/annotation/processing/ForyStructProcessorTest.java b/java/fory-annotation-processor/src/test/java/org/apache/fory/annotation/processing/ForyStructProcessorTest.java index a5c5095fad..d5faa619aa 100644 --- a/java/fory-annotation-processor/src/test/java/org/apache/fory/annotation/processing/ForyStructProcessorTest.java +++ b/java/fory-annotation-processor/src/test/java/org/apache/fory/annotation/processing/ForyStructProcessorTest.java @@ -42,6 +42,7 @@ import org.apache.fory.context.MetaReadContext; import org.apache.fory.context.MetaWriteContext; import org.apache.fory.exception.DeserializationException; +import org.apache.fory.exception.ForyException; import org.apache.fory.exception.SerializationException; import org.apache.fory.meta.FieldInfo; import org.apache.fory.meta.TypeDef; @@ -216,6 +217,43 @@ public void testStaticRegisteredConstructor() throws Exception { } } + @Test + public void testStaticCtorCopyBackref() throws Exception { + CompilationResult result = + compile( + "test.StaticCtorBackref", + "package test;\n" + + "import org.apache.fory.annotation.ForyConstructor;\n" + + "import org.apache.fory.annotation.ForyStruct;\n" + + "@ForyStruct public class StaticCtorBackref {\n" + + " public Object self;\n" + + " @ForyConstructor(\"self\")\n" + + " public StaticCtorBackref(Object self) { this.self = self; }\n" + + "}\n"); + Assert.assertTrue(result.success, result.diagnostics()); + String generatedSource = + result.generatedSource("test/StaticCtorBackref_ForyNativeSerializer.java"); + Assert.assertTrue(generatedSource.contains("beginConstructorCopy(copyContext, value)")); + Assert.assertTrue(generatedSource.contains("checkNoConstructorCopyBackrefs(")); + try (URLClassLoader loader = result.classLoader()) { + Class type = loader.loadClass("test.StaticCtorBackref"); + Object value = type.getConstructor(Object.class).newInstance((Object) null); + setField(type, value, "self", value); + Fory fory = + Fory.builder() + .withXlang(false) + .withClassLoader(loader) + .withCodegen(false) + .withRefTracking(true) + .withRefCopy(true) + .requireClassRegistration(false) + .build(); + Object serializer = fory.getTypeResolver().getTypeInfo(type).getSerializer(); + Assert.assertTrue(serializer instanceof StaticGeneratedStructSerializer); + Assert.assertThrows(ForyException.class, () -> fory.copy(value)); + } + } + @Test public void testForyDebugAnnotationEmitsGeneratedFieldTracing() throws Exception { CompilationResult result = diff --git a/java/fory-core/src/main/java/org/apache/fory/AbstractThreadSafeFory.java b/java/fory-core/src/main/java/org/apache/fory/AbstractThreadSafeFory.java index 9d9f427201..9f93cc21dc 100644 --- a/java/fory-core/src/main/java/org/apache/fory/AbstractThreadSafeFory.java +++ b/java/fory-core/src/main/java/org/apache/fory/AbstractThreadSafeFory.java @@ -21,7 +21,9 @@ import java.lang.reflect.Constructor; import java.util.function.Function; +import org.apache.fory.exception.ForyException; import org.apache.fory.resolver.TypeChecker; +import org.apache.fory.resolver.TypeInfo; import org.apache.fory.resolver.TypeResolver; import org.apache.fory.serializer.Serializer; import org.apache.fory.serializer.SerializerFactory; @@ -70,7 +72,26 @@ public void register(ForyModule module) { @Override public void registerConstructor( Class type, Constructor constructor, String... fieldNames) { - registerCallback(fory -> fory.registerConstructor(type, constructor, fieldNames)); + String[] copiedFieldNames = fieldNames.clone(); + registerCallback(fory -> fory.registerConstructor(type, constructor, copiedFieldNames)); + } + + protected static void checkRegisterConstructorAllowed(Fory fory, Class type) { + TypeResolver typeResolver = fory.getTypeResolver(); + if (typeResolver.isRegistrationFinished()) { + throw new ForyException( + "Cannot register class/serializer after registration has been frozen. Please register " + + "all classes before invoking top-level `serialize/deserialize/copy` methods of " + + "Fory."); + } + TypeInfo typeInfo = typeResolver.getTypeInfo(type, false); + if (typeInfo != null && typeInfo.getSerializer() != null) { + throw new ForyException( + "Cannot register constructor for " + + type.getName() + + " after its serializer has been created. Register constructors before calling " + + "`getSerializer`, `serialize`, `deserialize`, or `copy` for that type."); + } } public void registerUnion( diff --git a/java/fory-core/src/main/java/org/apache/fory/BaseFory.java b/java/fory-core/src/main/java/org/apache/fory/BaseFory.java index 0dc0cc0623..6f32c34ffd 100644 --- a/java/fory-core/src/main/java/org/apache/fory/BaseFory.java +++ b/java/fory-core/src/main/java/org/apache/fory/BaseFory.java @@ -94,10 +94,13 @@ public interface BaseFory { * Register an explicit constructor-to-field mapping for {@code type}. * *

    The constructor arguments are populated from {@code fieldNames} in order. This is useful for - * third-party classes that cannot annotate a constructor with {@code @ForyConstructor}. + * third-party classes that cannot annotate a constructor with {@code @ForyConstructor}. {@code + * fieldNames} must contain at least one field; ordinary no-argument constructors should not be + * registered because they do not need constructor-to-field binding. Java platform classes are + * owned by built-in serializers and cannot use explicit constructor binding. * - *

    Call this during setup before top-level serialization, deserialization, or copy operations - * start. + *

    Call this during setup before serializers for {@code type} are requested and before + * top-level serialization, deserialization, or copy operations start. */ void registerConstructor(Class type, Constructor constructor, String... fieldNames); diff --git a/java/fory-core/src/main/java/org/apache/fory/Fory.java b/java/fory-core/src/main/java/org/apache/fory/Fory.java index 695774ce5e..81b8eeabdb 100644 --- a/java/fory-core/src/main/java/org/apache/fory/Fory.java +++ b/java/fory-core/src/main/java/org/apache/fory/Fory.java @@ -216,6 +216,14 @@ public void register(ForyModule module) { public void registerConstructor( Class type, Constructor constructor, String... fieldNames) { checkRegisterAllowed(); + TypeInfo typeInfo = typeResolver.getTypeInfo(type, false); + if (typeInfo != null && typeInfo.getSerializer() != null) { + throw new ForyException( + "Cannot register constructor for " + + type.getName() + + " after its serializer has been created. Register constructors before calling " + + "`getSerializer`, `serialize`, `deserialize`, or `copy` for that type."); + } sharedRegistry.getObjectCreatorRegistry().registerConstructor(type, constructor, fieldNames); } diff --git a/java/fory-core/src/main/java/org/apache/fory/ThreadLocalFory.java b/java/fory-core/src/main/java/org/apache/fory/ThreadLocalFory.java index 0b00c719c5..2a2422332f 100644 --- a/java/fory-core/src/main/java/org/apache/fory/ThreadLocalFory.java +++ b/java/fory-core/src/main/java/org/apache/fory/ThreadLocalFory.java @@ -20,6 +20,7 @@ package org.apache.fory; import java.io.OutputStream; +import java.lang.reflect.Constructor; import java.nio.ByteBuffer; import java.util.Collections; import java.util.Map; @@ -79,10 +80,24 @@ private Fory currentFory() { @Override public void registerCallback(Consumer callback) { synchronized (callbackLock) { + synchronized (allFory) { + allFory.keySet().forEach(callback); + } factoryCallback = factoryCallback.andThen(callback); + } + } + + @Override + public void registerConstructor( + Class type, Constructor constructor, String... fieldNames) { + String[] copiedFieldNames = fieldNames.clone(); + Consumer callback = fory -> fory.registerConstructor(type, constructor, copiedFieldNames); + synchronized (callbackLock) { synchronized (allFory) { + allFory.keySet().forEach(fory -> checkRegisterConstructorAllowed(fory, type)); allFory.keySet().forEach(callback); } + factoryCallback = factoryCallback.andThen(callback); } } diff --git a/java/fory-core/src/main/java/org/apache/fory/annotation/ForyConstructor.java b/java/fory-core/src/main/java/org/apache/fory/annotation/ForyConstructor.java index 8ce7dd5ef2..ee5053434b 100644 --- a/java/fory-core/src/main/java/org/apache/fory/annotation/ForyConstructor.java +++ b/java/fory-core/src/main/java/org/apache/fory/annotation/ForyConstructor.java @@ -34,6 +34,9 @@ /** * Field names in constructor argument order. * + *

    At least one field must be listed. Ordinary no-argument constructors should not be annotated + * because they do not need constructor-to-field binding. + * *

    Every name must refer to one non-static serialized field declared by the target class or a * superclass. Duplicate field names in a class hierarchy are not bindable by this annotation. */ diff --git a/java/fory-core/src/main/java/org/apache/fory/builder/UnsafeCodegenSupport.java b/java/fory-core/src/main/java/org/apache/fory/builder/UnsafeCodegenSupport.java index 59390a2f04..b5ae3242cf 100644 --- a/java/fory-core/src/main/java/org/apache/fory/builder/UnsafeCodegenSupport.java +++ b/java/fory-core/src/main/java/org/apache/fory/builder/UnsafeCodegenSupport.java @@ -71,9 +71,9 @@ static String unsafeTypeName() { return "sun.misc.Unsafe"; } - static String unsafeInitCode() { + public static String unsafeInitCode() { checkUnsafeSupported(); - return "(sun.misc.Unsafe) " + UnsafeCodegenSupport.class.getName() + ".unsafe()"; + return "((sun.misc.Unsafe) " + UnsafeCodegenSupport.class.getName() + ".unsafe())"; } private static void checkUnsafeSupported() { diff --git a/java/fory-core/src/main/java/org/apache/fory/codegen/Expression.java b/java/fory-core/src/main/java/org/apache/fory/codegen/Expression.java index db3439023a..89090c6100 100644 --- a/java/fory-core/src/main/java/org/apache/fory/codegen/Expression.java +++ b/java/fory-core/src/main/java/org/apache/fory/codegen/Expression.java @@ -59,8 +59,8 @@ import java.util.Locale; import java.util.stream.Collectors; import java.util.stream.Stream; +import org.apache.fory.builder.UnsafeCodegenSupport; import org.apache.fory.platform.JdkVersion; -import org.apache.fory.platform.internal._JDKAccess; import org.apache.fory.reflect.ObjectCreator; import org.apache.fory.reflect.ObjectCreators; import org.apache.fory.reflect.ReflectionUtils; @@ -1520,7 +1520,7 @@ public ExprCode doGenCode(CodegenContext ctx) { functionName = "newInstance"; args = ""; } else { - target = ctx.type(_JDKAccess.class) + ".unsafe()"; + target = UnsafeCodegenSupport.unsafeInitCode(); functionName = "allocateInstance"; args = clzName + ".class"; } diff --git a/java/fory-core/src/main/java/org/apache/fory/memory/LittleEndian.java b/java/fory-core/src/main/java/org/apache/fory/memory/LittleEndian.java index d3facaee67..a2105ef17e 100644 --- a/java/fory-core/src/main/java/org/apache/fory/memory/LittleEndian.java +++ b/java/fory-core/src/main/java/org/apache/fory/memory/LittleEndian.java @@ -75,7 +75,9 @@ public static long getInt64(byte[] o, int index) { if (AndroidSupport.IS_ANDROID) { return MemoryOps.getInt64(o, index); } - long v = UNSAFE.getLong(o, BYTE_ARRAY_OFFSET + index); + // Unsafe object offsets are long. Keep the cast so JDK8-compiled bytecode calls + // getLong(Object, long) when the artifact runs on JDK9+. + long v = UNSAFE.getLong(o, (long) BYTE_ARRAY_OFFSET + index); return NativeByteOrder.IS_LITTLE_ENDIAN ? v : Long.reverseBytes(v); } @@ -87,6 +89,7 @@ public static void putInt64(byte[] o, int index, long value) { if (!NativeByteOrder.IS_LITTLE_ENDIAN) { value = Long.reverseBytes(value); } - UNSAFE.putLong(o, BYTE_ARRAY_OFFSET + index, value); + // See getInt64: the cast controls the Unsafe method descriptor in bytecode. + UNSAFE.putLong(o, (long) BYTE_ARRAY_OFFSET + index, value); } } diff --git a/java/fory-core/src/main/java/org/apache/fory/pool/ThreadPoolFory.java b/java/fory-core/src/main/java/org/apache/fory/pool/ThreadPoolFory.java index e3544aa106..2bba3c3e74 100644 --- a/java/fory-core/src/main/java/org/apache/fory/pool/ThreadPoolFory.java +++ b/java/fory-core/src/main/java/org/apache/fory/pool/ThreadPoolFory.java @@ -20,6 +20,7 @@ package org.apache.fory.pool; import java.io.OutputStream; +import java.lang.reflect.Constructor; import java.nio.ByteBuffer; import java.util.concurrent.Semaphore; import java.util.concurrent.atomic.AtomicInteger; @@ -154,6 +155,20 @@ public void registerCallback(Consumer callback) { } } + @Override + public void registerConstructor( + Class type, Constructor constructor, String... fieldNames) { + String[] copiedFieldNames = fieldNames.clone(); + synchronized (callbackLock) { + for (Fory fory : pooledFory) { + checkRegisterConstructorAllowed(fory, type); + } + for (Fory fory : pooledFory) { + fory.registerConstructor(type, constructor, copiedFieldNames); + } + } + } + @Override public R execute(Function action) { PooledEntry entry = acquire(); diff --git a/java/fory-core/src/main/java/org/apache/fory/reflect/ObjectCreatorRegistry.java b/java/fory-core/src/main/java/org/apache/fory/reflect/ObjectCreatorRegistry.java index 9932484af5..a1e520727a 100644 --- a/java/fory-core/src/main/java/org/apache/fory/reflect/ObjectCreatorRegistry.java +++ b/java/fory-core/src/main/java/org/apache/fory/reflect/ObjectCreatorRegistry.java @@ -45,7 +45,8 @@ public void registerConstructor( Class type, Constructor constructor, String... fieldNames) { ObjectCreators.ConstructorMatch match = ObjectCreators.explicitConstructor(type, constructor, fieldNames.clone(), "registered"); + ObjectCreator objectCreator = ObjectCreators.createObjectCreator(type, match); constructorMatches.put(type, match); - objectCreatorCache.put(type, ObjectCreators.createObjectCreator(type, match)); + objectCreatorCache.put(type, objectCreator); } } diff --git a/java/fory-core/src/main/java/org/apache/fory/reflect/ObjectCreators.java b/java/fory-core/src/main/java/org/apache/fory/reflect/ObjectCreators.java index 439525dcd9..7784b3670f 100644 --- a/java/fory-core/src/main/java/org/apache/fory/reflect/ObjectCreators.java +++ b/java/fory-core/src/main/java/org/apache/fory/reflect/ObjectCreators.java @@ -19,6 +19,7 @@ package org.apache.fory.reflect; +import java.io.Serializable; import java.lang.invoke.MethodHandle; import java.lang.invoke.MethodHandles.Lookup; import java.lang.invoke.MethodType; @@ -34,6 +35,7 @@ import java.util.Map; import java.util.Set; import org.apache.fory.annotation.ForyConstructor; +import org.apache.fory.annotation.Internal; import org.apache.fory.collection.ClassValueCache; import org.apache.fory.collection.Tuple2; import org.apache.fory.exception.ForyException; @@ -74,6 +76,8 @@ public class ObjectCreators { private static final ClassValueCache> cache = ClassValueCache.newClassKeySoftCache(8); + private static final ClassValueCache> objectStreamCache = + ClassValueCache.newClassKeySoftCache(8); /** * Returns an optimized ObjectCreator for the given type. @@ -96,6 +100,18 @@ public static ObjectCreator getObjectCreator(TypeResolver typeResolver, C return typeResolver.getSharedRegistry().getObjectCreatorRegistry().getObjectCreator(type); } + /** + * Returns the creator used by Java ObjectStream-compatible serializers. + * + *

    ObjectStream serializers reconstruct an empty instance before applying stream fields. + * Serializable class constructors and constructor mappings from {@link ForyConstructor} or {@code + * BaseFory.registerConstructor} are not semantically valid for this path. + */ + @Internal + public static ObjectCreator getObjectStreamCreator(Class type) { + return (ObjectCreator) objectStreamCache.get(type, () -> createObjectStreamCreator(type)); + } + static ObjectCreator createObjectCreator( Class type, ConstructorMatch registeredConstructor) { if (RecordUtils.isRecord(type)) { @@ -126,6 +142,21 @@ static ObjectCreator createObjectCreator( return new DeclaredNoArgCtrObjectCreator<>(type); } + private static ObjectCreator createObjectStreamCreator(Class type) { + if (AndroidSupport.IS_ANDROID) { + Constructor noArgConstructor = ReflectionUtils.getNoArgConstructor(type); + if (noArgConstructor != null) { + return new ReflectiveNoArgCtrObjectCreator<>(type, noArgConstructor); + } + return new UnsupportedObjectCreator<>( + type, "Android cannot create " + type + " without an accessible no-arg constructor"); + } + if (GraalvmSupport.IN_GRAALVM_NATIVE_IMAGE || JdkVersion.MAJOR_VERSION >= 25) { + return new UnsafeObjectCreator<>(type); + } + return new ParentNoArgCtrObjectCreator<>(type); + } + static final class ConstructorMatch { private final Constructor constructor; private final String[] fieldNames; @@ -156,7 +187,7 @@ private static ConstructorMatch explicitConstructor( ForyConstructor annotation = null; for (Constructor constructor : type.getDeclaredConstructors()) { ForyConstructor currentAnnotation = constructor.getAnnotation(ForyConstructor.class); - if (currentAnnotation == null) { + if (currentAnnotation == null || isCompilerGeneratedConstructor(constructor)) { continue; } if (annotatedConstructor != null) { @@ -172,12 +203,35 @@ private static ConstructorMatch explicitConstructor( type, (Constructor) annotatedConstructor, annotation.value(), "@ForyConstructor"); } + private static boolean isCompilerGeneratedConstructor(Constructor constructor) { + if (constructor.isSynthetic()) { + return true; + } + Class[] parameterTypes = constructor.getParameterTypes(); + return parameterTypes.length > 0 + && "kotlin.jvm.internal.DefaultConstructorMarker" + .equals(parameterTypes[parameterTypes.length - 1].getName()); + } + static ConstructorMatch explicitConstructor( Class type, Constructor constructor, String[] fieldNames, String source) { + if (isJavaPlatformType(type)) { + throw new ForyException( + source + + " constructor binding is not supported for Java platform type " + + type.getName()); + } if (constructor.getDeclaringClass() != type) { throw new ForyException( source + " constructor " + constructor + " does not belong to " + type); } + if (fieldNames.length == 0) { + throw new ForyException( + source + + " constructor " + + constructor + + " must map at least one field. Leave ordinary no-arg constructors unbound."); + } if (fieldNames.length != constructor.getParameterCount()) { throw new ForyException( source @@ -203,6 +257,10 @@ static ConstructorMatch explicitConstructor( return match; } + private static boolean isJavaPlatformType(Class type) { + return type.getName().startsWith("java."); + } + private static Field[] fieldsByExplicitNames( Class type, Constructor constructor, String[] names, String source) { List fields = new ArrayList<>(); @@ -531,16 +589,7 @@ private static Constructor createSerializationConstructor(Class type) reflectionFactoryClass.getMethod( "newConstructorForSerialization", Class.class, Constructor.class); } - Constructor parentConstructor = findPublicNoArgConstructor(type); - if (parentConstructor == null) { - parentConstructor = Object.class.getDeclaredConstructor(); - } else { - try { - parentConstructor.newInstance(); - } catch (Throwable ignored) { - parentConstructor = Object.class.getDeclaredConstructor(); - } - } + Constructor parentConstructor = findSerializationConstructor(type); return (Constructor) newConstructorForSerializationMethod.invoke(reflectionFactory, type, parentConstructor); } catch (Throwable e) { @@ -549,20 +598,38 @@ private static Constructor createSerializationConstructor(Class type) } } - private static Constructor findPublicNoArgConstructor(Class type) { + private static Constructor findSerializationConstructor(Class type) + throws NoSuchMethodException { Class current = type.getSuperclass(); - while (current != null && current != Object.class) { - try { - Constructor constructor = current.getDeclaredConstructor(); - if (Modifier.isPublic(constructor.getModifiers())) { - return constructor; - } - } catch (NoSuchMethodException ignored) { - // Continue searching - } + // Java ObjectStream reconstruction skips every Serializable class constructor and invokes + // only the first non-Serializable superclass no-arg constructor. + while (current != null && Serializable.class.isAssignableFrom(current)) { current = current.getSuperclass(); } - return null; + if (current == null) { + current = Object.class; + } + Constructor constructor = current.getDeclaredConstructor(); + if (!validSerializationConstructor(type, current, constructor)) { + throw new ForyException( + "First non-Serializable superclass " + + current.getName() + + " does not expose a valid no-arg constructor for " + + type); + } + return constructor; + } + + private static boolean validSerializationConstructor( + Class type, Class constructorClass, Constructor constructor) { + int modifiers = constructor.getModifiers(); + if (Modifier.isPublic(modifiers) || Modifier.isProtected(modifiers)) { + return true; + } + if (Modifier.isPrivate(modifiers)) { + return false; + } + return ReflectionUtils.getPackage(type).equals(ReflectionUtils.getPackage(constructorClass)); } @Override diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java index 48404bccde..59515e120a 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java @@ -34,7 +34,6 @@ import java.lang.reflect.Method; import java.math.BigDecimal; import java.math.BigInteger; -import java.net.URL; import java.nio.ByteBuffer; import java.nio.charset.Charset; import java.time.Duration; @@ -62,7 +61,6 @@ import java.util.Set; import java.util.SortedMap; import java.util.SortedSet; -import java.util.StringTokenizer; import java.util.TimeZone; import java.util.TreeMap; import java.util.TreeSet; @@ -106,7 +104,6 @@ import org.apache.fory.logging.LoggerFactory; import org.apache.fory.memory.ByteBufferUtil; import org.apache.fory.memory.MemoryBuffer; -import org.apache.fory.memory.MemoryUtils; import org.apache.fory.meta.ClassSpec; import org.apache.fory.meta.EncodedMetaString; import org.apache.fory.meta.Encoders; @@ -141,7 +138,6 @@ import org.apache.fory.serializer.Shareable; import org.apache.fory.serializer.SqlTimeSerializers; import org.apache.fory.serializer.TimeSerializers; -import org.apache.fory.serializer.URLSerializer; import org.apache.fory.serializer.UnknownClass; import org.apache.fory.serializer.UnknownClass.UnknownEmptyStruct; import org.apache.fory.serializer.UnknownClass.UnknownStruct; @@ -1475,15 +1471,6 @@ public Class getSerializerClass(Class cls, boolean code return TimeSerializers.ZoneIdSerializer.class; } else if (TimeZone.class.isAssignableFrom(cls)) { return TimeSerializers.TimeZoneSerializer.class; - } else if (cls == URL.class) { - return URLSerializer.class; - } else if (cls == StringTokenizer.class) { - if (!MemoryUtils.JDK_COLLECTION_FIELD_ACCESS) { - throw new UnsupportedOperationException( - "StringTokenizer serialization requires JDK internal field access. On JDK25+, open " - + "java.base/java.lang.invoke to org.apache.fory.core."); - } - return Serializers.StringTokenizerSerializer.class; } else if (ByteBuffer.class.isAssignableFrom(cls)) { return BufferSerializers.ByteBufferSerializer.class; } diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/AbstractObjectSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/AbstractObjectSerializer.java index 520070ccd3..da195c6c38 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/AbstractObjectSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/AbstractObjectSerializer.java @@ -19,11 +19,16 @@ package org.apache.fory.serializer; +import java.lang.reflect.Array; import java.lang.reflect.Field; import java.lang.reflect.Modifier; import java.util.ArrayList; import java.util.Arrays; +import java.util.IdentityHashMap; import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; import org.apache.fory.Fory; import org.apache.fory.collection.IntArray; @@ -77,6 +82,8 @@ public abstract class AbstractObjectSerializer extends Serializer { private final Class[] objectCreatorConstructorFieldDeclaringClasses; private final Class[] objectCreatorConstructorFieldTypes; private final boolean[] objectCreatorConstructorFieldFinal; + private volatile int[] copyConstructorFieldIndexes; + private volatile boolean[] copyConstructorFieldMask; private SerializationFieldInfo[] fieldInfos; private RecordInfo copyRecordInfo; @@ -549,6 +556,7 @@ private static Object readContainerFieldValueRef( RefReader refReader, SerializationFieldInfo fieldInfo, MemoryBuffer buffer) { + trackConstructorRefRead(readContext, buffer); int nextReadRefId = refReader.tryPreserveRefId(buffer); if (nextReadRefId >= Fory.NOT_NULL_VALUE_FLAG) { Object value; @@ -920,12 +928,12 @@ private T copyConstructorObject(CopyContext copyContext, T originObj) { if (fieldInfos == null) { fieldInfos = buildFieldsInfo(); } - int[] constructorFieldIndexes = buildConstructorFieldIndexes(fieldInfos); - boolean[] constructorFieldMask = - buildConstructorFieldMask(fieldInfos.length, constructorFieldIndexes); - checkNoSelfConstructorField(originObj, fieldInfos, constructorFieldMask); + int[] constructorFieldIndexes = copyConstructorFieldIndexes(fieldInfos); + boolean[] constructorFieldMask = copyConstructorFieldMask(fieldInfos, constructorFieldIndexes); + Object pendingMarker = beginConstructorCopy(copyContext, originObj); Object[] fieldValues = copyFieldValues(copyContext, originObj, fieldInfos, constructorFieldMask, true); + checkNoConstructorCopyBackrefs(fieldValues, constructorFieldIndexes, pendingMarker); T newObj = objectCreator.newInstanceWithArguments( constructorArgs( @@ -935,16 +943,23 @@ private T copyConstructorObject(CopyContext copyContext, T originObj) { return newObj; } - private void checkNoSelfConstructorField( - T originObj, SerializationFieldInfo[] fieldInfos, boolean[] constructorFieldMask) { - for (int i = 0; i < fieldInfos.length; i++) { - if (constructorFieldMask[i] && !fieldInfos[i].isPrimitiveField) { - Object fieldValue = fieldInfos[i].fieldAccessor.getObject(originObj); - if (fieldValue == originObj) { - throwConstructorCycle(type); - } - } + private int[] copyConstructorFieldIndexes(SerializationFieldInfo[] fieldInfos) { + int[] indexes = copyConstructorFieldIndexes; + if (indexes == null) { + indexes = buildConstructorFieldIndexes(fieldInfos); + copyConstructorFieldIndexes = indexes; + } + return indexes; + } + + private boolean[] copyConstructorFieldMask( + SerializationFieldInfo[] fieldInfos, int[] constructorFieldIndexes) { + boolean[] mask = copyConstructorFieldMask; + if (mask == null) { + mask = buildConstructorFieldMask(fieldInfos.length, constructorFieldIndexes); + copyConstructorFieldMask = mask; } + return mask; } private Object[] copyFieldValues(CopyContext copyContext, T originObj) { @@ -1288,6 +1303,119 @@ protected final void checkNoUnresolvedReadRef(ReadContext readContext) { checkNoUnresolvedReadRef(readContext, type); } + protected final Object beginConstructorCopy(CopyContext copyContext, Object originObj) { + if (!copyContext.copyTrackingRef()) { + return null; + } + Object pendingMarker = new ConstructorCopyPending(); + copyContext.reference(originObj, pendingMarker); + return pendingMarker; + } + + protected final void checkNoConstructorCopyBackrefs( + Object[] fieldValues, int[] constructorFieldIndexes, Object pendingMarker) { + if (pendingMarker == null) { + return; + } + IdentityHashMap seen = null; + for (int constructorFieldIndex : constructorFieldIndexes) { + if (constructorFieldIndex >= 0) { + seen = + checkNoConstructorCopyBackref( + fieldValues[constructorFieldIndex], pendingMarker, type, seen); + } + } + } + + private static IdentityHashMap checkNoConstructorCopyBackref( + Object fieldValue, + Object pendingMarker, + Class type, + IdentityHashMap seen) { + if (fieldValue == null || isConstructorBackrefLeaf(fieldValue.getClass())) { + return seen; + } + if (fieldValue == pendingMarker) { + throwConstructorCycle(type); + } + if (seen == null) { + seen = new IdentityHashMap<>(); + } else if (seen.containsKey(fieldValue)) { + return seen; + } + seen.put(fieldValue, Boolean.TRUE); + Class fieldClass = fieldValue.getClass(); + if (fieldClass.isArray()) { + if (!fieldClass.getComponentType().isPrimitive()) { + int length = Array.getLength(fieldValue); + for (int i = 0; i < length; i++) { + seen = checkNoConstructorCopyBackref(Array.get(fieldValue, i), pendingMarker, type, seen); + } + } + return seen; + } + if (fieldValue instanceof Optional) { + Optional optional = (Optional) fieldValue; + if (optional.isPresent()) { + seen = checkNoConstructorCopyBackref(optional.get(), pendingMarker, type, seen); + } + return seen; + } + if (fieldValue instanceof AtomicReference) { + return checkNoConstructorCopyBackref( + ((AtomicReference) fieldValue).get(), pendingMarker, type, seen); + } + if (fieldValue instanceof Iterable) { + for (Object element : (Iterable) fieldValue) { + seen = checkNoConstructorCopyBackref(element, pendingMarker, type, seen); + } + return seen; + } + if (fieldValue instanceof Map) { + for (Map.Entry entry : ((Map) fieldValue).entrySet()) { + seen = checkNoConstructorCopyBackref(entry.getKey(), pendingMarker, type, seen); + seen = checkNoConstructorCopyBackref(entry.getValue(), pendingMarker, type, seen); + } + return seen; + } + if (fieldValue instanceof Map.Entry) { + Map.Entry entry = (Map.Entry) fieldValue; + seen = checkNoConstructorCopyBackref(entry.getKey(), pendingMarker, type, seen); + return checkNoConstructorCopyBackref(entry.getValue(), pendingMarker, type, seen); + } + if (isJdkClass(fieldClass)) { + return seen; + } + for (Descriptor descriptor : Descriptor.getDescriptors(fieldClass)) { + if (descriptor.getRawType().isPrimitive() || descriptor.getField() == null) { + continue; + } + Object value = FieldAccessor.createAccessor(descriptor.getField()).getObject(fieldValue); + seen = checkNoConstructorCopyBackref(value, pendingMarker, type, seen); + } + return seen; + } + + private static final class ConstructorCopyPending {} + + private static boolean isConstructorBackrefLeaf(Class fieldClass) { + return fieldClass.isPrimitive() + || fieldClass.isEnum() + || fieldClass == Class.class + || fieldClass == String.class + || Number.class.isAssignableFrom(fieldClass) + || fieldClass == Boolean.class + || fieldClass == Character.class; + } + + private static boolean isJdkClass(Class fieldClass) { + String name = fieldClass.getName(); + return name.startsWith("java.") + || name.startsWith("javax.") + || name.startsWith("jdk.") + || name.startsWith("sun."); + } + public static void checkNoUnresolvedReadRef(ReadContext readContext, Class type) { if (consumeSelfRef(readContext)) { throwConstructorCycle(type); diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/CompatibleCollectionArrayReader.java b/java/fory-core/src/main/java/org/apache/fory/serializer/CompatibleCollectionArrayReader.java index c125edd722..ad8e22ef62 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/CompatibleCollectionArrayReader.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/CompatibleCollectionArrayReader.java @@ -307,7 +307,9 @@ private static Object readTracking( int elementTypeId, Class targetType) { RefReader refReader = readContext.getRefReader(); - int nextReadRefId = refReader.tryPreserveRefId(readContext.getBuffer()); + MemoryBuffer buffer = readContext.getBuffer(); + AbstractObjectSerializer.trackConstructorRefRead(readContext, buffer); + int nextReadRefId = refReader.tryPreserveRefId(buffer); if (nextReadRefId >= Fory.NOT_NULL_VALUE_FLAG) { Object value = readNotNull(readContext, readMode, arrayTypeId, elementTypeId, targetType); refReader.setReadRef(nextReadRefId, value); diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/ExceptionSerializers.java b/java/fory-core/src/main/java/org/apache/fory/serializer/ExceptionSerializers.java index 547b97c62c..1fba01ea5b 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/ExceptionSerializers.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/ExceptionSerializers.java @@ -144,10 +144,8 @@ public T read(ReadContext readContext) { if (stackTrace != null) { obj.setStackTrace(stackTrace); } - if (cause != null) { - obj.initCause(cause); - } - addSuppressedExceptions(obj, suppressedExceptions); + ThrowableAccessors.setCause(obj, cause); + ThrowableAccessors.setSuppressedExceptions(obj, suppressedExceptions); readAndSetFields(readContext, obj, slotsSerializers, config); return obj; } @@ -571,15 +569,38 @@ private static boolean containsPendingThrowable(Throwable throwable, Set suppressedExceptions) { + SUPPRESSED_ACCESSOR.putObject( + throwable, + suppressedExceptions.isEmpty() + ? DEFAULT_SUPPRESSED_EXCEPTIONS + : new ArrayList<>(suppressedExceptions)); + } } private static final class PendingThrowable extends Throwable { diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectStreamSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectStreamSerializer.java index 3a0e868e93..c3865b2f85 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectStreamSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectStreamSerializer.java @@ -65,7 +65,7 @@ import org.apache.fory.platform.AndroidSupport; import org.apache.fory.platform.GraalvmSupport; import org.apache.fory.platform.internal._JDKAccess; -import org.apache.fory.reflect.ObjectCreator; +import org.apache.fory.reflect.ObjectCreators; import org.apache.fory.resolver.ClassResolver; import org.apache.fory.resolver.TypeInfo; import org.apache.fory.resolver.TypeResolver; @@ -162,17 +162,17 @@ CompatibleLayerSerializerBase getReadSerializer( /** * Safe wrapper for ObjectStreamClass.lookup that handles GraalVM native image limitations. In * GraalVM native image, ObjectStreamClass.lookup may fail for certain classes like Throwable due - * to missing SerializationConstructorAccessor. This method catches such errors and returns null, - * allowing the serializer to use alternative approaches like Unsafe.allocateInstance. + * to missing SerializationConstructorAccessor. This method catches such errors and returns null + * so the serializer can use its constructor-bypassing object creator. */ private static ObjectStreamClass safeObjectStreamClassLookup(Class type) { if (GraalvmSupport.IN_GRAALVM_NATIVE_IMAGE) { try { return ObjectStreamClass.lookup(type); } catch (Throwable e) { - // In GraalVM native image, ObjectStreamClass.lookup may fail for certain classes - // due to missing SerializationConstructorAccessor. We catch this and return null - // to allow fallback to Unsafe-based object creation. + // In GraalVM native image, ObjectStreamClass.lookup may fail for certain classes due to + // missing SerializationConstructorAccessor. Returning null keeps stream reconstruction on + // the serializer-owned object creator path. LOG.warn( "ObjectStreamClass.lookup failed for {} in GraalVM native image: {}", type.getName(), @@ -186,7 +186,7 @@ private static ObjectStreamClass safeObjectStreamClassLookup(Class type) { } public ObjectStreamSerializer(TypeResolver typeResolver, Class type) { - super(typeResolver, type, createObjectStreamCreator(typeResolver, type)); + super(typeResolver, type, ObjectCreators.getObjectStreamCreator(type)); if (!Serializable.class.isAssignableFrom(type)) { throw new IllegalArgumentException( String.format("Class %s should implement %s.", type, Serializable.class)); @@ -215,12 +215,6 @@ public ObjectStreamSerializer(TypeResolver typeResolver, Class type) { slotsInfos = slotsInfoList.toArray(new SlotInfo[0]); } - /** Creates an ObjectCreator for Java ObjectStream-compatible reconstruction. */ - private static ObjectCreator createObjectStreamCreator( - TypeResolver typeResolver, Class type) { - return typeResolver.getObjectCreator(type); - } - @Override public void write(WriteContext writeContext, Object value) { MemoryBuffer buffer = writeContext.getBuffer(); diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/PlatformStringUtils.java b/java/fory-core/src/main/java/org/apache/fory/serializer/PlatformStringUtils.java index e2335af192..57e2e434e6 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/PlatformStringUtils.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/PlatformStringUtils.java @@ -151,7 +151,9 @@ static long getBytesLong(byte[] bytes, int byteIndex) { | ((long) bytes[byteIndex + 7] & 0xff); } } - return UNSAFE.getLong(bytes, BYTE_ARRAY_OFFSET + byteIndex); + // Unsafe object offsets are long. Keep the cast so JDK8-compiled bytecode calls + // getLong(Object, long) when the artifact runs on JDK9+. + return UNSAFE.getLong(bytes, (long) BYTE_ARRAY_OFFSET + byteIndex); } static char getBytesChar(byte[] bytes, int byteIndex) { @@ -162,7 +164,7 @@ static char getBytesChar(byte[] bytes, int byteIndex) { return (char) (((bytes[byteIndex] & 0xff) << 8) | (bytes[byteIndex + 1] & 0xff)); } } - return UNSAFE.getChar(bytes, BYTE_ARRAY_OFFSET + byteIndex); + return UNSAFE.getChar(bytes, (long) BYTE_ARRAY_OFFSET + byteIndex); } static void copyCharsToBytes( diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/Serializers.java b/java/fory-core/src/main/java/org/apache/fory/serializer/Serializers.java index 85a091516d..042e69e2d9 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/Serializers.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/Serializers.java @@ -30,8 +30,8 @@ import java.math.BigInteger; import java.net.URI; import java.nio.charset.Charset; +import java.util.Comparator; import java.util.Currency; -import java.util.StringTokenizer; import java.util.UUID; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; @@ -48,13 +48,13 @@ import org.apache.fory.context.CopyContext; import org.apache.fory.context.ReadContext; import org.apache.fory.context.WriteContext; +import org.apache.fory.exception.ForyException; import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.memory.MemoryUtils; import org.apache.fory.meta.TypeDef; import org.apache.fory.platform.AndroidSupport; import org.apache.fory.platform.GraalvmSupport; import org.apache.fory.platform.internal._JDKAccess; -import org.apache.fory.reflect.FieldAccessor; import org.apache.fory.reflect.ReflectionUtils; import org.apache.fory.resolver.ClassResolver; import org.apache.fory.resolver.TypeInfo; @@ -596,88 +596,6 @@ public StringBuffer read(ReadContext readContext) { } } - public static final class StringTokenizerSerializer extends Serializer - implements Shareable { - public StringTokenizerSerializer(Config config) { - super(config, StringTokenizer.class); - } - - @Override - public void write(WriteContext writeContext, StringTokenizer value) { - checkStringTokenizerAccess(); - MemoryBuffer buffer = writeContext.getBuffer(); - writeContext.writeRef(Accessors.STR.getObject(value)); - writeContext.writeRef(Accessors.DELIMITERS.getObject(value)); - buffer.writeBoolean(Accessors.RET_DELIMS.getBoolean(value)); - buffer.writeVarInt32(Accessors.CURRENT_POSITION.getInt(value)); - buffer.writeVarInt32(Accessors.NEW_POSITION.getInt(value)); - buffer.writeBoolean(Accessors.DELIMS_CHANGED.getBoolean(value)); - } - - @Override - public StringTokenizer read(ReadContext readContext) { - checkStringTokenizerAccess(); - String str = (String) readContext.readRef(); - String delimiters = (String) readContext.readRef(); - boolean retDelims = readContext.getBuffer().readBoolean(); - StringTokenizer tokenizer = new StringTokenizer(str, delimiters, retDelims); - restoreState(readContext.getBuffer(), tokenizer); - return tokenizer; - } - - @Override - public StringTokenizer copy(CopyContext copyContext, StringTokenizer value) { - checkStringTokenizerAccess(); - StringTokenizer tokenizer = - new StringTokenizer( - (String) Accessors.STR.getObject(value), - (String) Accessors.DELIMITERS.getObject(value), - Accessors.RET_DELIMS.getBoolean(value)); - Accessors.CURRENT_POSITION.putInt(tokenizer, Accessors.CURRENT_POSITION.getInt(value)); - Accessors.NEW_POSITION.putInt(tokenizer, Accessors.NEW_POSITION.getInt(value)); - Accessors.DELIMS_CHANGED.putBoolean(tokenizer, Accessors.DELIMS_CHANGED.getBoolean(value)); - return tokenizer; - } - - private static void restoreState(MemoryBuffer buffer, StringTokenizer tokenizer) { - Accessors.CURRENT_POSITION.putInt(tokenizer, buffer.readVarInt32()); - Accessors.NEW_POSITION.putInt(tokenizer, buffer.readVarInt32()); - Accessors.DELIMS_CHANGED.putBoolean(tokenizer, buffer.readBoolean()); - } - - private static void checkStringTokenizerAccess() { - if (!MemoryUtils.JDK_COLLECTION_FIELD_ACCESS) { - throw stringTokenizerAccessError(); - } - } - - private static UnsupportedOperationException stringTokenizerAccessError() { - return new UnsupportedOperationException( - "StringTokenizer serialization requires JDK internal field access. On JDK25+, open " - + "java.base/java.lang.invoke to org.apache.fory.core."); - } - - private static final class Accessors { - private static final FieldAccessor CURRENT_POSITION = - FieldAccessor.createAccessor( - ReflectionUtils.getField(StringTokenizer.class, "currentPosition")); - private static final FieldAccessor NEW_POSITION = - FieldAccessor.createAccessor( - ReflectionUtils.getField(StringTokenizer.class, "newPosition")); - private static final FieldAccessor STR = - FieldAccessor.createAccessor(ReflectionUtils.getField(StringTokenizer.class, "str")); - private static final FieldAccessor DELIMITERS = - FieldAccessor.createAccessor( - ReflectionUtils.getField(StringTokenizer.class, "delimiters")); - private static final FieldAccessor RET_DELIMS = - FieldAccessor.createAccessor( - ReflectionUtils.getField(StringTokenizer.class, "retDelims")); - private static final FieldAccessor DELIMS_CHANGED = - FieldAccessor.createAccessor( - ReflectionUtils.getField(StringTokenizer.class, "delimsChanged")); - } - } - public static final class AtomicBooleanSerializer extends Serializer implements Shareable { @@ -905,15 +823,34 @@ public Object read(ReadContext readContext) { } } + public static final class ReverseComparatorSerializer extends ImmutableSerializer + implements Shareable { + private static final byte ORIGINAL_REPLACE_RESOLVE_PAYLOAD = 0; + + public ReverseComparatorSerializer(Config config) { + super(config, (Class) Comparator.reverseOrder().getClass()); + } + + @Override + public void write(WriteContext writeContext, Comparator value) { + writeContext.getBuffer().writeByte(ORIGINAL_REPLACE_RESOLVE_PAYLOAD); + } + + @Override + public Comparator read(ReadContext readContext) { + byte payload = readContext.getBuffer().readByte(); + if (payload != ORIGINAL_REPLACE_RESOLVE_PAYLOAD) { + throw new ForyException("Unexpected reverse comparator payload flag " + payload); + } + return Comparator.reverseOrder(); + } + } + public static void registerDefaultSerializers(TypeResolver resolver) { Config config = resolver.getConfig(); resolver.registerInternalSerializer(Class.class, new ClassSerializer(config)); resolver.registerInternalSerializer(StringBuilder.class, new StringBuilderSerializer(config)); resolver.registerInternalSerializer(StringBuffer.class, new StringBufferSerializer(config)); - // Keep this internal type id reserved even when JDK collection internals are not open; - // otherwise payloads written with access enabled decode later collection ids incorrectly. - resolver.registerInternalSerializer( - StringTokenizer.class, new StringTokenizerSerializer(config)); resolver.registerInternalSerializer(BigInteger.class, new BigIntegerSerializer(config)); resolver.registerInternalSerializer(BigDecimal.class, new DecimalSerializer(config)); resolver.registerInternalSerializer(AtomicBoolean.class, new AtomicBooleanSerializer(config)); @@ -926,5 +863,7 @@ public static void registerDefaultSerializers(TypeResolver resolver) { resolver.registerInternalSerializer(Pattern.class, new RegexSerializer(config)); resolver.registerInternalSerializer(UUID.class, new UUIDSerializer(config)); resolver.registerInternalSerializer(Object.class, new EmptyObjectSerializer(config)); + resolver.registerInternalSerializer( + Comparator.reverseOrder().getClass(), new ReverseComparatorSerializer(config)); } } diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/StaticGeneratedStructSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/StaticGeneratedStructSerializer.java index f348d64c2f..a13c7934e1 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/StaticGeneratedStructSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/StaticGeneratedStructSerializer.java @@ -340,9 +340,6 @@ protected final Object copyConstructorFieldValue( Object originObject, Object fieldValue, SerializationFieldInfo fieldInfo) { - if (fieldValue == originObject) { - throwConstructorCycle(type); - } return copyFieldValue(copyContext, fieldValue, fieldInfo); } diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/CollectionSerializers.java b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/CollectionSerializers.java index df3e3a14f4..db5dcf80f0 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/CollectionSerializers.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/CollectionSerializers.java @@ -620,12 +620,18 @@ public Collection newCollection(ReadContext readContext) { public Collection newCollection(CopyContext copyContext, Collection originCollection) { assert !config.isXlang(); if (!MemoryUtils.JDK_COLLECTION_FIELD_ACCESS || GraalvmSupport.IN_GRAALVM_NATIVE_IMAGE) { + if (JdkVersion.MAJOR_VERSION >= 25) { + throw unsupportedJdk25SetFromMap(null); + } return Collections.newSetFromMap(new HashMap(originCollection.size())); } Map map = (Map) JvmSetFromMapAccess.MAP_ACCESSOR.getObject(originCollection); MapLikeSerializer mapSerializer = (MapLikeSerializer) typeResolver.getSerializer(map.getClass()); + if (JdkVersion.MAJOR_VERSION >= 25 && !mapSerializer.supportCodegenHook) { + throw unsupportedJdk25SetFromMap(map.getClass()); + } Map newMap = mapSerializer.newMap(copyContext, map); return Collections.newSetFromMap(newMap); } @@ -636,6 +642,9 @@ public Collection onCollectionWrite(WriteContext writeContext, Set value) { Map map; TypeInfo typeInfo; if (!MemoryUtils.JDK_COLLECTION_FIELD_ACCESS || GraalvmSupport.IN_GRAALVM_NATIVE_IMAGE) { + if (JdkVersion.MAJOR_VERSION >= 25) { + throw unsupportedJdk25SetFromMap(null); + } HashMap source = new HashMap<>(value.size()); for (Object element : value) { source.put(element, Boolean.TRUE); @@ -648,15 +657,9 @@ public Collection onCollectionWrite(WriteContext writeContext, Set value) { } MapLikeSerializer mapSerializer = (MapLikeSerializer) typeInfo.getSerializer(); // The legacy payload restores Collections$SetFromMap by writing its final JDK fields. - // JDK25 zero-Unsafe mode cannot do that, so emit the public-constructor shape instead. + // JDK25 zero-Unsafe mode cannot do that, so unsupported backing maps must fail before write. if (JdkVersion.MAJOR_VERSION >= 25 && !mapSerializer.supportCodegenHook) { - HashMap source = new HashMap<>(value.size()); - for (Object element : value) { - source.put(element, Boolean.TRUE); - } - map = source; - typeInfo = typeResolver.getTypeInfo(HashMap.class); - mapSerializer = (MapLikeSerializer) typeInfo.getSerializer(); + throw unsupportedJdk25SetFromMap(map.getClass()); } typeResolver.writeTypeInfo(writeContext, typeInfo); if (mapSerializer.supportCodegenHook) { @@ -669,6 +672,16 @@ public Collection onCollectionWrite(WriteContext writeContext, Set value) { return EMPTY_COLLECTION_STUB; } } + + private static UnsupportedOperationException unsupportedJdk25SetFromMap(Class mapType) { + String mapDescription = mapType == null ? "an inaccessible backing map" : mapType.getName(); + return new UnsupportedOperationException( + "JDK25+ zero-Unsafe mode cannot serialize Collections.newSetFromMap backed by " + + mapDescription + + " because that would require hidden final JDK field restoration. Use a backing " + + "map with public-constructor serialization support or register a custom " + + "serializer."); + } } public static final class ConcurrentHashMapKeySetViewSerializer diff --git a/java/fory-core/src/main/java25/org/apache/fory/builder/UnsafeCodegenSupport.java b/java/fory-core/src/main/java25/org/apache/fory/builder/UnsafeCodegenSupport.java index 44aad39bb0..ae3d46c34e 100644 --- a/java/fory-core/src/main/java25/org/apache/fory/builder/UnsafeCodegenSupport.java +++ b/java/fory-core/src/main/java25/org/apache/fory/builder/UnsafeCodegenSupport.java @@ -39,7 +39,7 @@ static String unsafeTypeName() { throw unsupported(); } - static String unsafeInitCode() { + public static String unsafeInitCode() { throw unsupported(); } diff --git a/java/fory-core/src/main/java25/org/apache/fory/memory/MemoryBuffer.java b/java/fory-core/src/main/java25/org/apache/fory/memory/MemoryBuffer.java index 86827aaeee..ecd6fb1745 100644 --- a/java/fory-core/src/main/java25/org/apache/fory/memory/MemoryBuffer.java +++ b/java/fory-core/src/main/java25/org/apache/fory/memory/MemoryBuffer.java @@ -55,8 +55,8 @@ * part as separate class, and use composition in this class. In this way, all fields can be final * and access will be much faster. * - *

    Warning: The instance of this class should not be hold on graalvm build time, the heap unsafe - * offset are not correct in runtime since graalvm will change array base offset. + *

    Warning: The instance of this class should not be held at GraalVM build time; build-time heap + * buffers do not represent the runtime heap layout. * *

    Note(chaokunyang): Buffer operations are very common, and jvm inline and branch elimination is * not reliable even in c2 compiler, so we try to inline and avoid checks as we can manually. jvm @@ -122,8 +122,8 @@ public final class MemoryBuffer { final ForyStreamReader streamReader; // Android branches in this class are intentional method-boundary exits. - // Do not delete them or fold them into the JVM Unsafe path: each branch must make exactly one - // MemoryOps call, while MemoryOps owns Android heap index math and reader/writer updates. + // Do not delete them or fold them into the JVM path: each branch must make exactly one MemoryOps + // call, while MemoryOps owns Android heap index math and reader/writer updates. /** * Creates a new memory buffer that represents the memory of the byte array. @@ -716,7 +716,7 @@ public void get(int index, byte[] dst) { public void get(int index, byte[] dst, int offset, int length) { final byte[] heapMemory = this.heapMemory; if (heapMemory != null) { - // System.arraycopy faster for some jdk than Unsafe. + // Keep heap-to-heap bulk copies on the JDK intrinsic path. System.arraycopy(heapMemory, heapOffset + index, dst, offset, length); } else { final long pos = address + index; @@ -802,7 +802,7 @@ public void put(int index, byte[] src) { public void put(int index, byte[] src, int offset, int length) { final byte[] heapMemory = this.heapMemory; if (heapMemory != null) { - // System.arraycopy faster for some jdk than Unsafe. + // Keep heap-to-heap bulk copies on the JDK intrinsic path. System.arraycopy(src, offset, heapMemory, heapOffset + index, length); } else { final long pos = address + index; @@ -2923,7 +2923,7 @@ public long readVarUint36Small() { if (AndroidSupport.IS_ANDROID) { return MemoryOps.readVarUint36Small(this); } - // Android exits above. Keep JVM small-varint bulk reads as raw Unsafe loads instead of calling + // Android exits above. Keep JVM small-varint bulk reads as direct bulk loads instead of calling // `_unsafeGet*` helpers; those helpers carry Android/endian branches and can break inlining. // Duplicate and manual inline for performance. // noinspection Duplicates diff --git a/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties b/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties index e0549368c0..9f01c6e257 100644 --- a/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties +++ b/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties @@ -884,5 +884,4 @@ Args=--initialize-at-build-time=org.apache.fory.annotation.ForyField$Dynamic,\ org.apache.fory.shaded.org.codehaus.janino.util.ClassFile$StackMapTableAttribute$StackMapFrame,\ org.apache.fory.shaded.org.codehaus.janino.util.ClassFile$StackMapTableAttribute$UninitializedVariableInfo,\ org.apache.fory.shaded.org.codehaus.janino.util.ClassFile$StackMapTableAttribute$VerificationTypeInfo,\ - org.apache.fory.shaded.org.codehaus.janino.util.Traverser,\ - sun.misc.Unsafe + org.apache.fory.shaded.org.codehaus.janino.util.Traverser diff --git a/java/fory-core/src/test/java/org/apache/fory/ForyTest.java b/java/fory-core/src/test/java/org/apache/fory/ForyTest.java index ce93278584..11b3671203 100644 --- a/java/fory-core/src/test/java/org/apache/fory/ForyTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/ForyTest.java @@ -73,6 +73,7 @@ import org.apache.fory.serializer.ExceptionSerializers; import org.apache.fory.serializer.ObjectSerializer; import org.apache.fory.serializer.Serializer; +import org.apache.fory.serializer.Serializers; import org.apache.fory.test.bean.BeanA; import org.apache.fory.test.bean.Struct; import org.apache.fory.type.Descriptor; @@ -96,6 +97,18 @@ public void typedDeserializeRejectsOutOfBandRootHeaderWithoutBuffers() { assertThrows(IllegalArgumentException.class, () -> fory.deserialize(bytes, Integer.class)); } + @Test + public void testReverseComparatorSerializer() { + Fory fory = Fory.builder().withXlang(false).requireClassRegistration(false).build(); + Comparator comparator = Comparator.reverseOrder(); + Serializer serializer = + fory.getTypeResolver().getTypeInfo(comparator.getClass()).getSerializer(); + assertTrue(serializer instanceof Serializers.ReverseComparatorSerializer); + Object roundTrip = fory.deserialize(fory.serialize(comparator)); + Assert.assertSame(roundTrip, Comparator.reverseOrder()); + Assert.assertSame(fory.copy(comparator), comparator); + } + @Test(dataProvider = "crossLanguageReferenceTrackingConfig") public void primitivesTest(boolean referenceTracking, boolean xlang) { Fory fory1 = diff --git a/java/fory-core/src/test/java/org/apache/fory/JpmsOptionalClassLoadingTest.java b/java/fory-core/src/test/java/org/apache/fory/JpmsOptionalClassLoadingTest.java index ee7971e7fe..0d952e7072 100644 --- a/java/fory-core/src/test/java/org/apache/fory/JpmsOptionalClassLoadingTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/JpmsOptionalClassLoadingTest.java @@ -36,6 +36,10 @@ public void testBuildWithoutJavaSqlModule() throws Exception { if (JdkVersion.MAJOR_VERSION < 9) { throw new SkipException("Skip on jdk" + JdkVersion.MAJOR_VERSION); } + if (JdkVersion.MAJOR_VERSION >= 25) { + throw new SkipException( + "JDK25+ optional-module coverage runs in integration_tests/jpms_tests with packaged MR-JAR"); + } Process process = new ProcessBuilder( TestUtils.javaCommand( diff --git a/java/fory-core/src/test/java/org/apache/fory/ThreadSafeForyTest.java b/java/fory-core/src/test/java/org/apache/fory/ThreadSafeForyTest.java index 47fe199d7c..42aeff6634 100644 --- a/java/fory-core/src/test/java/org/apache/fory/ThreadSafeForyTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/ThreadSafeForyTest.java @@ -25,6 +25,7 @@ import static org.testng.Assert.assertSame; import static org.testng.Assert.assertTrue; +import java.lang.reflect.Constructor; import java.nio.ByteBuffer; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CountDownLatch; @@ -51,14 +52,14 @@ public class ThreadSafeForyTest extends ForyTestBase { @Test - public void testBuildThreadSafeForyUsesThreadPoolFory() { + public void testBuildThreadSafeForyPool() { ThreadSafeFory fory = Fory.builder().withXlang(false).requireClassRegistration(false).buildThreadSafeFory(); assertTrue(fory instanceof ThreadPoolFory); } @Test - public void testThreadSafeBuildersAssignGeneratedNames() { + public void testThreadSafeBuilderNames() { ThreadSafeFory threadSafe = Fory.builder().withXlang(false).requireClassRegistration(false).buildThreadSafeFory(); ThreadSafeFory threadLocal = @@ -86,7 +87,7 @@ public void testThreadSafeBuildersAssignGeneratedNames() { } @Test - public void testFunctionFactoryConstructorsUseBuilderProvidedClassLoader() { + public void testFactoryConstructorsClassLoader() { ClassLoader custom = new ClassLoader(ClassLoader.getSystemClassLoader()) {}; ThreadLocalFory threadLocal = new ThreadLocalFory( @@ -99,7 +100,72 @@ public void testFunctionFactoryConstructorsUseBuilderProvidedClassLoader() { } @Test - public void testThreadSafeRuntimesShareRegistryAcrossRawForyInstances() throws Exception { + public void testThreadLocalCtorRegFailure() throws Exception { + ThreadLocalFory fory = + Fory.builder() + .withXlang(false) + .withCodegen(false) + .requireClassRegistration(false) + .buildThreadLocalFory(); + fory.execute( + f -> { + f.getSerializer(ThreadSafeCtorBean.class); + return null; + }); + Constructor constructor = + ThreadSafeCtorBean.class.getDeclaredConstructor(int.class, String.class); + Assert.assertThrows( + ForyException.class, + () -> fory.registerConstructor(ThreadSafeCtorBean.class, constructor, "age", "name")); + assertFutureThreadCreator(fory, false); + } + + @Test + public void testThreadLocalCtorRegCopies() throws Exception { + ThreadLocalFory fory = + Fory.builder() + .withXlang(false) + .withCodegen(false) + .requireClassRegistration(false) + .buildThreadLocalFory(); + Constructor constructor = + ThreadSafeCtorBean.class.getDeclaredConstructor(int.class, String.class); + String[] fieldNames = {"age", "name"}; + fory.registerConstructor(ThreadSafeCtorBean.class, constructor, fieldNames); + fieldNames[0] = "name"; + fieldNames[1] = "age"; + assertFutureThreadCreator(fory, true); + } + + @Test + public void testThreadPoolCtorRegPreflight() throws Exception { + ThreadSafeFory fory = + Fory.builder() + .withXlang(false) + .withCodegen(false) + .requireClassRegistration(false) + .buildThreadSafeForyPool(2); + fory.execute( + f -> { + f.getSerializer(ThreadSafeCtorBean.class); + return null; + }); + Constructor constructor = + ThreadSafeCtorBean.class.getDeclaredConstructor(int.class, String.class); + Assert.assertThrows( + ForyException.class, + () -> fory.registerConstructor(ThreadSafeCtorBean.class, constructor, "age", "name")); + assertEquals( + fory.execute( + f -> + f.getTypeResolver() + .getObjectCreator(ThreadSafeCtorBean.class) + .hasConstructorFields()), + Boolean.FALSE); + } + + @Test + public void testThreadSafeRuntimesShareRegistry() throws Exception { ThreadLocalFory threadLocal = Fory.builder().withXlang(false).requireClassRegistration(false).buildThreadLocalFory(); AtomicReference threadLocalRegistry1 = new AtomicReference<>(); @@ -316,7 +382,7 @@ public void testSerializeWithMetaShare() throws InterruptedException { } @Test - public void testThreadLocalMetaShareWithPerThreadMetaContexts() throws InterruptedException { + public void testThreadLocalMetaShare() throws InterruptedException { ThreadSafeFory fory = Fory.builder() .withXlang(false) @@ -380,7 +446,7 @@ public void testSerializeDeserializeWithType() { } @Test - public void testDeserializeByteBufferPreservesPositionAndLimit() { + public void testByteBufferPositionLimit() { Fory writer = Fory.builder().withXlang(false).requireClassRegistration(false).build(); String value = "thread-safe-byte-buffer"; byte[] payload = writer.serialize(value); @@ -423,12 +489,48 @@ private static ByteBuffer[] byteBufferViews(byte[] payload) { return new ByteBuffer[] {heap, heapReadOnly, direct, directReadOnly}; } + private static void assertFutureThreadCreator(ThreadSafeFory fory, boolean constructorFields) + throws Exception { + AtomicReference result = new AtomicReference<>(); + AtomicReference error = new AtomicReference<>(); + Thread thread = + new Thread( + () -> { + try { + result.set( + fory.execute( + f -> + f.getTypeResolver() + .getObjectCreator(ThreadSafeCtorBean.class) + .hasConstructorFields())); + } catch (Throwable t) { + error.set(t); + } + }); + thread.start(); + thread.join(); + if (error.get() != null) { + throw new AssertionError(error.get()); + } + assertEquals(result.get(), Boolean.valueOf(constructorFields)); + } + private static byte[] wrapWithPadding(byte[] payload) { byte[] bytes = new byte[payload.length + 6]; System.arraycopy(payload, 0, bytes, 3, payload.length); return bytes; } + public static final class ThreadSafeCtorBean { + final int age; + final String name; + + private ThreadSafeCtorBean(int age, String name) { + this.age = age; + this.name = name; + } + } + @Data static class Foo { int f1; @@ -505,7 +607,7 @@ public void testSerializerRegister() { } @Test - public void testRegisterAfterSerializeThrowsException() { + public void testRegisterAfterSerializeThrows() { ThreadSafeFory fory = Fory.builder().withXlang(false).requireClassRegistration(true).buildThreadLocalFory(); fory.register(BeanA.class); @@ -514,7 +616,7 @@ public void testRegisterAfterSerializeThrowsException() { } @Test - public void testRegisterAfterSerializeThrowsExceptionWithFory() { + public void testForyRegisterAfterSerializeThrows() { Fory fory = Fory.builder().withXlang(false).requireClassRegistration(true).build(); fory.register(BeanA.class); fory.serialize("ok"); @@ -522,7 +624,7 @@ public void testRegisterAfterSerializeThrowsExceptionWithFory() { } @Test - public void testRegisterAfterSerializeThrowsExceptionWithForyPool() { + public void testPoolRegisterAfterSerializeThrows() { ThreadSafeFory fory = Fory.builder().withXlang(false).requireClassRegistration(true).buildThreadSafeForyPool(2); fory.register(BeanA.class); diff --git a/java/fory-core/src/test/java/org/apache/fory/memory/MemoryBufferTest.java b/java/fory-core/src/test/java/org/apache/fory/memory/MemoryBufferTest.java index 58fe730cc1..a7f2f062f0 100644 --- a/java/fory-core/src/test/java/org/apache/fory/memory/MemoryBufferTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/memory/MemoryBufferTest.java @@ -85,7 +85,7 @@ public void testBufferWrite() { } @Test - public void testFromDirectByteBufferRejectsHeapBuffer() { + public void testDirectBufferRejectsHeap() { assertThrows( IllegalArgumentException.class, () -> MemoryBuffer.fromDirectByteBuffer(ByteBuffer.allocate(8), 8, null)); @@ -612,7 +612,7 @@ public void testWriteVarUInt32() { } @Test - public void testReadVarUInt32RejectsMalformedFifthByte() { + public void testReadVarUInt32RejectsFifthByte() { byte[] malformed = new byte[] {(byte) 0x80, (byte) 0x80, (byte) 0x80, (byte) 0x80, 0x10}; assertThrows(IllegalArgumentException.class, () -> MemoryUtils.wrap(malformed).readVarUInt32()); assertThrows( diff --git a/java/fory-core/src/test/java/org/apache/fory/reflect/ObjectCreatorsTest.java b/java/fory-core/src/test/java/org/apache/fory/reflect/ObjectCreatorsTest.java index bda955b4c4..7bdae7cf71 100644 --- a/java/fory-core/src/test/java/org/apache/fory/reflect/ObjectCreatorsTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/reflect/ObjectCreatorsTest.java @@ -22,6 +22,7 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; +import java.lang.reflect.Constructor; import java.nio.charset.StandardCharsets; import java.util.concurrent.ArrayBlockingQueue; import org.apache.fory.TestUtils; @@ -66,6 +67,19 @@ public void testAndroidObjectCreators() throws Exception { Assert.assertEquals(process.waitFor(), 0, output); } + @Test + public void testFailedCtorRegNotPublished() throws Exception { + if (JdkVersion.MAJOR_VERSION < 9) { + return; + } + ObjectCreatorRegistry registry = new ObjectCreatorRegistry(); + Constructor constructor = String.class.getDeclaredConstructor(byte[].class, byte.class); + Assert.assertThrows( + ForyException.class, + () -> registry.registerConstructor(String.class, constructor, "value", "coder")); + Assert.assertFalse(registry.getObjectCreator(String.class).hasConstructorFields()); + } + private static String readFully(InputStream inputStream) throws IOException { ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); byte[] buffer = new byte[1024]; diff --git a/java/fory-core/src/test/java/org/apache/fory/serializer/ObjectSerializerTest.java b/java/fory-core/src/test/java/org/apache/fory/serializer/ObjectSerializerTest.java index 92f7d91fd4..15f0e2c490 100644 --- a/java/fory-core/src/test/java/org/apache/fory/serializer/ObjectSerializerTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/serializer/ObjectSerializerTest.java @@ -28,6 +28,9 @@ import java.io.InputStream; import java.lang.reflect.Constructor; import java.nio.charset.StandardCharsets; +import java.util.AbstractList; +import java.util.ArrayList; +import java.util.List; import lombok.Data; import org.apache.fory.Fory; import org.apache.fory.ForyTestBase; @@ -35,12 +38,17 @@ import org.apache.fory.annotation.ForyConstructor; import org.apache.fory.annotation.ForyField; import org.apache.fory.builder.CodecUtils; +import org.apache.fory.context.CopyContext; +import org.apache.fory.context.ReadContext; +import org.apache.fory.context.WriteContext; import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.memory.MemoryUtils; import org.apache.fory.meta.TypeDef; import org.apache.fory.platform.AndroidSupport; import org.apache.fory.platform.JdkVersion; +import org.apache.fory.resolver.RefMode; import org.apache.fory.test.bean.Cyclic; +import org.apache.fory.type.Types; import org.apache.fory.util.Preconditions; import org.testng.Assert; import org.testng.annotations.Test; @@ -240,6 +248,65 @@ public static final class ConstructorBackrefChild { private ConstructorBackrefRoot root; } + public static final class ConstructorCustomRoot { + private final ConstructorCustomHolder holder; + + @ForyConstructor("holder") + public ConstructorCustomRoot(ConstructorCustomHolder holder) { + this.holder = holder; + } + } + + public static final class ConstructorCustomHolder { + private final String label; + private ConstructorCustomRoot root; + + public ConstructorCustomHolder(String label) { + this.label = label; + } + } + + public static final class ConstructorCustomHolderSerializer + extends Serializer { + public ConstructorCustomHolderSerializer(Fory fory) { + super(fory.getConfig(), ConstructorCustomHolder.class, true, false); + } + + @Override + public void write(WriteContext writeContext, ConstructorCustomHolder value) { + throw new UnsupportedOperationException(); + } + + @Override + public ConstructorCustomHolder read(ReadContext readContext) { + throw new UnsupportedOperationException(); + } + + @Override + public ConstructorCustomHolder copy(CopyContext copyContext, ConstructorCustomHolder value) { + return new ConstructorCustomHolder(value.label); + } + } + + public static final class ConstructorContainerBackrefRoot extends AbstractList { + private List self; + + @ForyConstructor("self") + public ConstructorContainerBackrefRoot(List self) { + this.self = self; + } + + @Override + public Object get(int index) { + return self.get(index); + } + + @Override + public int size() { + return self == null ? 0 : self.size(); + } + } + public static final class RegisteredCtorBean { @ForyField(id = 0) private final String name; @@ -253,6 +320,21 @@ private RegisteredCtorBean(int age, String name) { } } + public static final class EmptyConstructorBinding { + private int id; + + @ForyConstructor({}) + public EmptyConstructorBinding() {} + + private EmptyConstructorBinding(int id) { + this.id = id; + } + } + + public static final class EmptyRegisteredCtorBean { + public EmptyRegisteredCtorBean() {} + } + public static final class FinalNoArgBean { private final int id; private final String name; @@ -428,6 +510,59 @@ public void testRegisterConstructor() throws Exception { } } + @Test + public void testRegisterConstructorAfterSerializer() throws Exception { + Constructor constructor = + RegisteredCtorBean.class.getDeclaredConstructor(int.class, String.class); + Fory fory = + Fory.builder() + .withXlang(false) + .withRefTracking(true) + .withCodegen(false) + .requireClassRegistration(false) + .build(); + fory.getSerializer(RegisteredCtorBean.class); + Assert.assertThrows( + org.apache.fory.exception.ForyException.class, + () -> fory.registerConstructor(RegisteredCtorBean.class, constructor, "age", "name")); + } + + @Test + public void testEmptyRegisteredCtorRejected() throws Exception { + Constructor constructor = + EmptyRegisteredCtorBean.class.getDeclaredConstructor(); + Fory fory = + Fory.builder() + .withXlang(false) + .withRefTracking(true) + .withCodegen(false) + .requireClassRegistration(false) + .build(); + Assert.assertThrows( + org.apache.fory.exception.ForyException.class, + () -> fory.registerConstructor(EmptyRegisteredCtorBean.class, constructor)); + } + + @Test + public void testEmptyAnnotatedCtorRejected() { + Assert.assertThrows( + org.apache.fory.exception.ForyException.class, + () -> + new ObjectSerializer<>( + newConstructorBindingFory(false).getTypeResolver(), EmptyConstructorBinding.class)); + Assert.assertThrows( + org.apache.fory.exception.ForyException.class, + () -> { + Fory fory = newConstructorBindingFory(true); + TypeDef typeDef = fory.getTypeResolver().getTypeDef(EmptyConstructorBinding.class, true); + new CompatibleSerializer<>( + fory.getTypeResolver(), EmptyConstructorBinding.class, typeDef); + }); + Assert.assertThrows( + org.apache.fory.exception.ForyException.class, + () -> newConstructorBindingFory(false).copy(new EmptyConstructorBinding(1))); + } + @Test public void testCtorInterveningRef() { Fory fory = @@ -544,6 +679,44 @@ public void testConstructorFieldBackrefRejected() { })); } + @Test + public void testContainerCtorBackrefRejected() { + if (JdkVersion.MAJOR_VERSION < 25) { + return; + } + ConstructorContainerBackrefRoot value = new ConstructorContainerBackrefRoot(null); + value.self = value; + Fory fory = + Fory.builder() + .withXlang(false) + .withRefTracking(true) + .withCodegen(false) + .requireClassRegistration(false) + .build(); + ObjectSerializer serializer = + new ObjectSerializer<>(fory.getTypeResolver(), ConstructorContainerBackrefRoot.class); + MemoryBuffer buffer = MemoryUtils.buffer(32); + withWriteContext( + fory, + buffer, + context -> { + context.writeRefOrNull(value); + serializer.write(context, value); + }); + Assert.assertThrows( + org.apache.fory.exception.ForyException.class, + () -> + withReadContext( + fory, + buffer, + context -> { + byte tag = context.readRefOrNull(); + Preconditions.checkArgument(tag == Fory.REF_VALUE_FLAG); + context.preserveRefId(); + return serializer.read(context); + })); + } + @Test public void testConstructorFieldCycle() { ConstructorCycle value = new ConstructorCycle("root"); @@ -604,7 +777,7 @@ public void testConstructorFieldCycleCodegen() { } @Test - public void testConstructorFieldCycleBeforeFinalCodegen() { + public void testCtorCycleBeforeFinalCodegen() { ConstructorCycleBeforeFinal value = new ConstructorCycleBeforeFinal("root"); value.next = value; Fory fory = @@ -648,7 +821,7 @@ public void testConstructorFieldCycleCompatible() { } @Test - public void testConstructorFieldCycleBeforeFinalCompatible() { + public void testCtorCycleBeforeFinalCompat() { ConstructorCycleBeforeFinal value = new ConstructorCycleBeforeFinal("root"); value.next = value; Fory fory = @@ -672,7 +845,7 @@ public void testConstructorFieldCycleBeforeFinalCompatible() { } @Test - public void testConstructorFieldCycleCompatibleNonCodegen() { + public void testCtorCycleCompatNoCodegen() { ConstructorCycleBeforeFinal value = new ConstructorCycleBeforeFinal("root"); value.next = value; Fory fory = @@ -693,7 +866,7 @@ public void testConstructorFieldCycleCompatibleNonCodegen() { } @Test - public void testConstructorFieldBackrefCompatibleRejected() { + public void testCtorBackrefCompatRejected() { if (JdkVersion.MAJOR_VERSION < 25) { return; } @@ -733,6 +906,132 @@ public void testConstructorFieldBackrefCompatibleRejected() { })); } + @Test + public void testContainerCtorBackrefCompatRejected() { + if (JdkVersion.MAJOR_VERSION < 25) { + return; + } + ConstructorContainerBackrefRoot value = new ConstructorContainerBackrefRoot(null); + value.self = value; + Fory fory = + Fory.builder() + .withXlang(false) + .withRefTracking(true) + .withCompatible(true) + .withCodegen(false) + .requireClassRegistration(false) + .build(); + TypeDef typeDef = + fory.getTypeResolver().getTypeDef(ConstructorContainerBackrefRoot.class, true); + CompatibleSerializer serializer = + new CompatibleSerializer<>( + fory.getTypeResolver(), ConstructorContainerBackrefRoot.class, typeDef); + MemoryBuffer buffer = MemoryUtils.buffer(32); + withWriteContext( + fory, + buffer, + context -> { + context.writeRefOrNull(value); + serializer.write(context, value); + }); + Assert.assertThrows( + org.apache.fory.exception.ForyException.class, + () -> + withReadContext( + fory, + buffer, + context -> { + byte tag = context.readRefOrNull(); + Preconditions.checkArgument(tag == Fory.REF_VALUE_FLAG); + context.preserveRefId(); + return serializer.read(context); + })); + } + + @Test + public void testCompatArrayCtorBackrefRejected() { + Object marker = new Object(); + Fory fory = + Fory.builder() + .withXlang(false) + .withRefTracking(true) + .withCodegen(false) + .requireClassRegistration(false) + .build(); + MemoryBuffer buffer = MemoryUtils.buffer(32); + withWriteContext( + fory, + buffer, + context -> { + context.writeRefOrNull(marker); + context.writeRefOrNull(marker); + }); + Assert.assertThrows( + org.apache.fory.exception.ForyException.class, + () -> + withReadContext( + fory, + buffer, + context -> { + byte tag = context.readRefOrNull(); + Preconditions.checkArgument(tag == Fory.REF_VALUE_FLAG); + context.preserveRefId(); + AbstractObjectSerializer.beginConstructorRef(context); + try { + Object value = + CompatibleCollectionArrayReader.read( + context, + RefMode.TRACKING, + CompatibleCollectionArrayReader.READ_ARRAY_TO_LIST, + Types.INT32_ARRAY, + Types.INT32, + List.class); + return AbstractObjectSerializer.ctorFieldValue( + context, value, ConstructorContainerBackrefRoot.class); + } finally { + AbstractObjectSerializer.endConstructorRef(context); + } + })); + } + + @Test(dataProvider = "foryCopyConfig") + public void testCtorBackrefCopyRejected(Fory fory) { + ConstructorBackrefChild child = new ConstructorBackrefChild(); + ConstructorBackrefRoot value = new ConstructorBackrefRoot(child); + child.root = value; + ObjectSerializer serializer = + new ObjectSerializer<>(fory.getTypeResolver(), ConstructorBackrefRoot.class); + Assert.assertThrows( + org.apache.fory.exception.ForyException.class, + () -> withCopyContext(fory, context -> serializer.copy(context, value))); + } + + @Test(dataProvider = "foryCopyConfig") + public void testContainerCtorBackrefCopyRejected(Fory fory) { + ConstructorContainerBackrefRoot value = new ConstructorContainerBackrefRoot(new ArrayList<>()); + value.self.add(value); + ObjectSerializer serializer = + new ObjectSerializer<>(fory.getTypeResolver(), ConstructorContainerBackrefRoot.class); + Assert.assertThrows( + org.apache.fory.exception.ForyException.class, + () -> withCopyContext(fory, context -> serializer.copy(context, value))); + } + + @Test(dataProvider = "foryCopyConfig") + public void testCtorCopyUsesSerializer(Fory fory) { + ConstructorCustomHolder holder = new ConstructorCustomHolder("custom"); + ConstructorCustomRoot value = new ConstructorCustomRoot(holder); + holder.root = value; + fory.registerSerializer( + ConstructorCustomHolder.class, new ConstructorCustomHolderSerializer(fory)); + ObjectSerializer serializer = + new ObjectSerializer<>(fory.getTypeResolver(), ConstructorCustomRoot.class); + ConstructorCustomRoot copy = withCopyContext(fory, context -> serializer.copy(context, value)); + assertNotSame(copy.holder, holder); + assertEquals(copy.holder.label, "custom"); + Assert.assertNull(copy.holder.root); + } + @Test(dataProvider = "foryCopyConfig") public void testConstructorFieldCycleCopy(Fory fory) { ConstructorCycle value = new ConstructorCycle("root"); @@ -782,6 +1081,16 @@ private static void assertInterveningRef(ConstructorInterveningRef value) { Assert.assertNotSame(value.second, value); } + private static Fory newConstructorBindingFory(boolean compatible) { + return Fory.builder() + .withXlang(false) + .withRefTracking(true) + .withCompatible(compatible) + .withCodegen(false) + .requireClassRegistration(false) + .build(); + } + @Data public static class A { Integer f1; @@ -810,7 +1119,7 @@ public void testSerialization() { } @Test - public void testAndroidObjectSerializerReflectionPaths() throws Exception { + public void testAndroidReflectionPaths() throws Exception { ProcessBuilder processBuilder = new ProcessBuilder(TestUtils.javaCommand(AndroidObjectSerializerProbe.class)) .redirectErrorStream(true); diff --git a/java/fory-core/src/test/java/org/apache/fory/serializer/ObjectStreamSerializerTest.java b/java/fory-core/src/test/java/org/apache/fory/serializer/ObjectStreamSerializerTest.java index 8e970cdec5..eaccede8c1 100644 --- a/java/fory-core/src/test/java/org/apache/fory/serializer/ObjectStreamSerializerTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/serializer/ObjectStreamSerializerTest.java @@ -29,6 +29,7 @@ import java.io.ObjectOutputStream; import java.io.ObjectStreamField; import java.io.Serializable; +import java.lang.reflect.Constructor; import java.lang.reflect.Method; import java.math.BigInteger; import java.net.Inet4Address; @@ -45,14 +46,17 @@ import lombok.EqualsAndHashCode; import org.apache.fory.Fory; import org.apache.fory.ForyTestBase; +import org.apache.fory.annotation.ForyConstructor; import org.apache.fory.config.ForyBuilder; import org.apache.fory.context.MetaReadContext; import org.apache.fory.context.MetaWriteContext; +import org.apache.fory.exception.ForyException; import org.apache.fory.exception.InsecureException; import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.resolver.SharedRegistry; import org.apache.fory.serializer.collection.CollectionSerializers; import org.apache.fory.serializer.collection.MapSerializers; +import org.apache.fory.serializer.otherpkg.PackageNoArgParent; import org.apache.fory.util.Preconditions; import org.testng.Assert; import org.testng.annotations.DataProvider; @@ -116,6 +120,217 @@ public void testDispatch(Fory fory) { serDeCheckSerializer(fory, o, "ObjectStreamSerializer"); } + public static class AnnotatedObjectStreamType implements Serializable { + String name; + int age; + transient boolean readObjectCalled; + + @ForyConstructor({"name", "age"}) + public AnnotatedObjectStreamType(String name, int age) { + this.name = name; + this.age = age; + } + + private void writeObject(ObjectOutputStream s) throws IOException { + s.defaultWriteObject(); + } + + private void readObject(ObjectInputStream s) throws IOException, ClassNotFoundException { + s.defaultReadObject(); + readObjectCalled = true; + } + } + + public static class RegisteredObjectStreamType implements Serializable { + String name; + int age; + transient boolean readObjectCalled; + + public RegisteredObjectStreamType(String name, int age) { + this.name = name; + this.age = age; + } + + private void writeObject(ObjectOutputStream s) throws IOException { + s.defaultWriteObject(); + } + + private void readObject(ObjectInputStream s) throws IOException, ClassNotFoundException { + s.defaultReadObject(); + readObjectCalled = true; + } + } + + public static class ThrowingNoArgObjectStreamType implements Serializable { + String name; + + public ThrowingNoArgObjectStreamType() { + throw new AssertionError("ObjectStream reconstruction must not invoke this constructor"); + } + + public ThrowingNoArgObjectStreamType(String name) { + this.name = name; + } + + private void writeObject(ObjectOutputStream s) throws IOException { + s.defaultWriteObject(); + } + + private void readObject(ObjectInputStream s) throws IOException, ClassNotFoundException { + s.defaultReadObject(); + } + } + + public static class SerializableParentCtorType implements Serializable { + static int noArgCalls; + String parentName; + + public SerializableParentCtorType() { + noArgCalls++; + } + + public SerializableParentCtorType(String parentName) { + this.parentName = parentName; + } + + private void writeObject(ObjectOutputStream s) throws IOException { + s.defaultWriteObject(); + } + + private void readObject(ObjectInputStream s) throws IOException, ClassNotFoundException { + s.defaultReadObject(); + } + } + + public static class SerializableChildCtorType extends SerializableParentCtorType { + String childName; + + public SerializableChildCtorType(String parentName, String childName) { + super(parentName); + this.childName = childName; + } + + private void writeObject(ObjectOutputStream s) throws IOException { + s.defaultWriteObject(); + } + + private void readObject(ObjectInputStream s) throws IOException, ClassNotFoundException { + s.defaultReadObject(); + } + } + + public static class PrivateNoArgParent { + private PrivateNoArgParent() {} + + public PrivateNoArgParent(String ignored) {} + } + + public static class InvalidCtorChild extends PrivateNoArgParent implements Serializable { + String name; + + public InvalidCtorChild(String name) { + super(name); + this.name = name; + } + + private void writeObject(ObjectOutputStream s) throws IOException { + s.defaultWriteObject(); + } + + private void readObject(ObjectInputStream s) throws IOException, ClassNotFoundException { + s.defaultReadObject(); + } + } + + public static class InvalidPackageCtorChild extends PackageNoArgParent implements Serializable { + String name; + + public InvalidPackageCtorChild(String name) { + super(name); + this.name = name; + } + + private void writeObject(ObjectOutputStream s) throws IOException { + s.defaultWriteObject(); + } + + private void readObject(ObjectInputStream s) throws IOException, ClassNotFoundException { + s.defaultReadObject(); + } + } + + @Test(dataProvider = "javaFory") + public void testConstructorMappingRead(Fory fory) throws NoSuchMethodException { + Constructor constructor = + RegisteredObjectStreamType.class.getDeclaredConstructor(String.class, int.class); + fory.registerConstructor(RegisteredObjectStreamType.class, constructor, "name", "age"); + + AnnotatedObjectStreamType annotated = + serDeCheckSerializer( + fory, new AnnotatedObjectStreamType("annotated", 1), "ObjectStreamSerializer"); + Assert.assertEquals(annotated.name, "annotated"); + Assert.assertEquals(annotated.age, 1); + Assert.assertTrue(annotated.readObjectCalled); + + RegisteredObjectStreamType registered = + serDeCheckSerializer( + fory, new RegisteredObjectStreamType("registered", 2), "ObjectStreamSerializer"); + Assert.assertEquals(registered.name, "registered"); + Assert.assertEquals(registered.age, 2); + Assert.assertTrue(registered.readObjectCalled); + + ThrowingNoArgObjectStreamType throwing = + serDeCheckSerializer( + fory, new ThrowingNoArgObjectStreamType("throwing"), "ObjectStreamSerializer"); + Assert.assertEquals(throwing.name, "throwing"); + } + + @Test(dataProvider = "javaFory") + public void testSerializableParentCtor(Fory fory) { + SerializableParentCtorType.noArgCalls = 0; + SerializableChildCtorType child = new SerializableChildCtorType("parent", "child"); + SerializableChildCtorType decoded = serDeCheckSerializer(fory, child, "ObjectStreamSerializer"); + Assert.assertEquals(decoded.parentName, "parent"); + Assert.assertEquals(decoded.childName, "child"); + Assert.assertEquals(SerializableParentCtorType.noArgCalls, 0); + } + + @Test(dataProvider = "javaFory") + public void testInvalidParentCtor(Fory fory) { + Assert.assertThrows(ForyException.class, () -> fory.serialize(new InvalidCtorChild("child"))); + Assert.assertThrows( + ForyException.class, () -> fory.serialize(new InvalidPackageCtorChild("child"))); + } + + @Test(dataProvider = "foryCopyConfig") + public void testConstructorMappingCopy(Fory fory) throws NoSuchMethodException { + Constructor constructor = + RegisteredObjectStreamType.class.getDeclaredConstructor(String.class, int.class); + fory.registerConstructor(RegisteredObjectStreamType.class, constructor, "name", "age"); + + AnnotatedObjectStreamType copy = fory.copy(new AnnotatedObjectStreamType("copy", 3)); + Assert.assertEquals(copy.name, "copy"); + Assert.assertEquals(copy.age, 3); + + RegisteredObjectStreamType registeredCopy = + fory.copy(new RegisteredObjectStreamType("registered-copy", 4)); + Assert.assertEquals(registeredCopy.name, "registered-copy"); + Assert.assertEquals(registeredCopy.age, 4); + + ThrowingNoArgObjectStreamType throwingCopy = + fory.copy(new ThrowingNoArgObjectStreamType("throwing-copy")); + Assert.assertEquals(throwingCopy.name, "throwing-copy"); + } + + @Test(dataProvider = "foryCopyConfig") + public void testSerializableParentCtorCopy(Fory fory) { + SerializableParentCtorType.noArgCalls = 0; + SerializableChildCtorType copy = fory.copy(new SerializableChildCtorType("parent", "child")); + Assert.assertEquals(copy.parentName, "parent"); + Assert.assertEquals(copy.childName, "child"); + Assert.assertEquals(SerializableParentCtorType.noArgCalls, 0); + } + @EqualsAndHashCode(callSuper = true) public static class WriteObjectTestClass2 extends WriteObjectTestClass { private final String data; @@ -495,7 +710,7 @@ public void testWriteObjectReplace(Fory fory) throws MalformedURLException { new ObjectStreamSerializer(fory.getTypeResolver(), WriteObjectTestClass4.class)); Assert.assertEquals( - serDeCheckSerializer(fory, new URL("http://test"), "URLSerializer"), + serDeCheckSerializer(fory, new URL("http://test"), "ReplaceResolve"), new URL("http://test")); WriteObjectTestClass4 testClassObj4 = new WriteObjectTestClass4(new char[] {'a', 'b'}); serDeCheckSerializer(fory, testClassObj4, "ObjectStreamSerializer"); diff --git a/java/fory-core/src/test/java/org/apache/fory/serializer/URLSerializerTest.java b/java/fory-core/src/test/java/org/apache/fory/serializer/URLSerializerTest.java index 5da5aa16c6..5035051921 100644 --- a/java/fory-core/src/test/java/org/apache/fory/serializer/URLSerializerTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/serializer/URLSerializerTest.java @@ -31,7 +31,7 @@ public class URLSerializerTest extends ForyTestBase { @Test(dataProvider = "javaFory") public void testDefaultWrite(Fory fory) throws MalformedURLException { Assert.assertEquals( - serDeCheckSerializer(fory, new URL("http://test"), "URLSerializer"), + serDeCheckSerializer(fory, new URL("http://test"), "ReplaceResolve"), new URL("http://test")); } diff --git a/java/fory-core/src/test/java/org/apache/fory/serializer/otherpkg/PackageNoArgParent.java b/java/fory-core/src/test/java/org/apache/fory/serializer/otherpkg/PackageNoArgParent.java new file mode 100644 index 0000000000..a2555858a2 --- /dev/null +++ b/java/fory-core/src/test/java/org/apache/fory/serializer/otherpkg/PackageNoArgParent.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.serializer.otherpkg; + +public class PackageNoArgParent { + PackageNoArgParent() {} + + public PackageNoArgParent(String ignored) {} +} diff --git a/java/fory-latest-jdk-tests/src/test/java/org/apache/fory/integration_tests/ImmutableCollectionSerializersTest.java b/java/fory-latest-jdk-tests/src/test/java/org/apache/fory/integration_tests/ImmutableCollectionSerializersTest.java index ed1afbff76..9d9878a445 100644 --- a/java/fory-latest-jdk-tests/src/test/java/org/apache/fory/integration_tests/ImmutableCollectionSerializersTest.java +++ b/java/fory-latest-jdk-tests/src/test/java/org/apache/fory/integration_tests/ImmutableCollectionSerializersTest.java @@ -21,6 +21,8 @@ import static org.apache.fory.integration_tests.TestUtils.serDeCheck; +import java.util.Collections; +import java.util.IdentityHashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -28,6 +30,9 @@ import org.apache.fory.Fory; import org.apache.fory.ThreadSafeFory; import org.apache.fory.annotation.ForyConstructor; +import org.apache.fory.exception.CopyException; +import org.apache.fory.exception.SerializationException; +import org.apache.fory.platform.JdkVersion; import org.apache.fory.test.bean.CollectionFields; import org.apache.fory.test.bean.MapFields; import org.testng.Assert; @@ -93,6 +98,24 @@ public void testImmutableMapStruct() { serDeCheck(fory, collectionFields); } + @Test + public void testSetFromMapIdentityJdk25() { + if (JdkVersion.MAJOR_VERSION < 25) { + return; + } + Fory fory = Fory.builder().withXlang(false).build(); + fory.register(IdentityHashMap.class); + Set set = Collections.newSetFromMap(new IdentityHashMap<>()); + set.add(new String("a")); + set.add(new String("a")); + Assert.assertEquals(set.size(), 2); + SerializationException exception = + Assert.expectThrows(SerializationException.class, () -> fory.serialize(set)); + Assert.assertTrue(exception.getCause() instanceof UnsupportedOperationException); + CopyException copyException = Assert.expectThrows(CopyException.class, () -> fory.copy(set)); + Assert.assertTrue(copyException.getCause() instanceof UnsupportedOperationException); + } + @Data public static class Pojo { List> data; diff --git a/java/pom.xml b/java/pom.xml index 358e8493e2..08d3cac451 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -241,7 +241,6 @@ ${maven.compiler.source} ${maven.compiler.target} - true diff --git a/kotlin/fory-kotlin-ksp/src/main/kotlin/org/apache/fory/kotlin/ksp/ForyKotlinSymbolProcessor.kt b/kotlin/fory-kotlin-ksp/src/main/kotlin/org/apache/fory/kotlin/ksp/ForyKotlinSymbolProcessor.kt index 8855925932..081b7941cb 100644 --- a/kotlin/fory-kotlin-ksp/src/main/kotlin/org/apache/fory/kotlin/ksp/ForyKotlinSymbolProcessor.kt +++ b/kotlin/fory-kotlin-ksp/src/main/kotlin/org/apache/fory/kotlin/ksp/ForyKotlinSymbolProcessor.kt @@ -113,6 +113,9 @@ internal fun constructorBindingError( fieldNames: List, targetName: String, ): String? { + if (fieldNames.isEmpty()) { + return "@ForyConstructor on $targetName must declare at least one field name" + } if (parameterNames.size != fieldNames.size) { return "@ForyConstructor on $targetName must declare one field name for each primary constructor parameter" } @@ -303,6 +306,13 @@ internal class ForyKotlinSymbolProcessor(private val environment: SymbolProcesso parseVarFields(declaration) ?: return null, ) } + if (constructorFields == null) { + logger.error( + "Kotlin KSP constructor-backed @ForyStruct requires @ForyConstructor field mappings", + primaryConstructor, + ) + return null + } val propertiesByName = declaration.getAllProperties().associateBy { it.simpleName.asString() } val fields = parseCtorFields(declaration, primaryConstructor, propertiesByName, constructorFields) @@ -375,17 +385,16 @@ internal class ForyKotlinSymbolProcessor(private val environment: SymbolProcesso declaration: KSClassDeclaration, primaryConstructor: KSFunctionDeclaration, propertiesByName: Map, - constructorFields: List?, + constructorFields: List, ): List? { val fields = mutableListOf() val foryIds = hashSetOf() var nextId = 0 - val explicitConstructor = constructorFields != null for ((index, parameter) in primaryConstructor.parameters.withIndex()) { val parameterName = parameter.name?.asString() ?: continue - val fieldName = constructorFields?.get(index) ?: parameterName + val fieldName = constructorFields[index] val property = propertiesByName[fieldName] - if (property == null || (!explicitConstructor && !parameter.isVal && !parameter.isVar)) { + if (property == null) { logger.error( "Constructor parameter $parameterName is not bound to an accessible schema property", parameter @@ -396,7 +405,7 @@ internal class ForyKotlinSymbolProcessor(private val environment: SymbolProcesso val parameterType = parameter.type.resolve() val fieldTypeName = kotlinSourceTypeName(fieldType) val parameterTypeName = kotlinSourceTypeName(parameterType) - if (explicitConstructor && fieldTypeName != parameterTypeName) { + if (fieldTypeName != parameterTypeName) { logger.error( "@ForyConstructor field $fieldName type $fieldTypeName must match primary constructor parameter $parameterName type $parameterTypeName", parameter, diff --git a/kotlin/fory-kotlin-ksp/src/main/kotlin/org/apache/fory/kotlin/ksp/KotlinSerializerSourceWriter.kt b/kotlin/fory-kotlin-ksp/src/main/kotlin/org/apache/fory/kotlin/ksp/KotlinSerializerSourceWriter.kt index afcee53408..995a786554 100644 --- a/kotlin/fory-kotlin-ksp/src/main/kotlin/org/apache/fory/kotlin/ksp/KotlinSerializerSourceWriter.kt +++ b/kotlin/fory-kotlin-ksp/src/main/kotlin/org/apache/fory/kotlin/ksp/KotlinSerializerSourceWriter.kt @@ -393,6 +393,7 @@ internal class KotlinSerializerSourceWriter(private val struct: KotlinSourceStru .append(struct.typeName) .append(" {\n") builder.append(" val fieldValues = arrayOfNulls(DESCRIPTORS.size)\n") + builder.append(" val pendingMarker = beginConstructorCopy(copyContext, value)\n") for (field in struct.fields) { builder.append(" if (hasField(constructorFieldBits!!, ").append(field.id).append(")) {\n") builder @@ -406,6 +407,9 @@ internal class KotlinSerializerSourceWriter(private val struct: KotlinSourceStru .append("]!!)\n") builder.append(" }\n") } + builder.append( + " checkNoConstructorCopyBackrefs(fieldValues, constructorFieldIds!!, pendingMarker)\n" + ) builder.append(" val copied = newConstructorObject(fieldValues)\n") builder.append(" copyContext.reference(value, copied)\n") for (field in struct.fields) { @@ -590,73 +594,116 @@ internal class KotlinSerializerSourceWriter(private val struct: KotlinSourceStru builder.append(" if (sameSchemaCompatible) {\n") builder.append(" return readSchemaConsistent(readContext)\n") builder.append(" }\n") - builder.append(" if (constructorFieldIds != null) {\n") - builder.append(" return readCompatibleConstructor(readContext)\n") - builder.append(" }\n") + if ( + struct.construction == KotlinStructConstruction.CONSTRUCTOR && + struct.fields.any { it.hasDefault } + ) { + builder.append(" return readCompatibleDefaultConstructor(readContext)\n") + builder.append(" }\n\n") + writeCompatibleDefaultConstructorRead() + return + } + if (struct.construction == KotlinStructConstruction.CONSTRUCTOR) { + builder.append(" if (constructorFieldIds != null) {\n") + builder.append(" return readCompatibleConstructor(readContext)\n") + builder.append(" }\n") + } if (struct.construction == KotlinStructConstruction.MUTABLE) { writeMutableCompatibleReadBody() builder.append(" }\n\n") return } - writePresenceVars() - writeLocalDeclarations() - builder.append(" for (i in remoteFields.indices) {\n") - builder.append(" val remoteField = remoteFields[i]\n") - builder.append(" when (remoteField.matchedId) {\n") + writeCompatibleValueReadBody(" ", constructorRefs = false) + builder.append(" }\n\n") + } + + private fun writeCompatibleDefaultConstructorRead() { + builder + .append(" private fun readCompatibleDefaultConstructor(readContext: ReadContext): ") + .append(struct.typeName) + .append(" {\n") + builder.append(" beginConstructorRef(readContext)\n") + builder.append(" try {\n") + writeCompatibleValueReadBody(" ", constructorRefs = true) + builder.append(" } finally {\n") + builder.append(" endConstructorRef(readContext)\n") + builder.append(" }\n") + builder.append(" }\n\n") + } + + private fun writeCompatibleValueReadBody(indent: String, constructorRefs: Boolean) { + writePresenceVars(indent) + writeLocalDeclarations(indent) + builder.append(indent).append("for (i in remoteFields.indices) {\n") + builder.append(indent).append(" val remoteField = remoteFields[i]\n") + builder.append(indent).append(" when (remoteField.matchedId) {\n") for (field in struct.fields) { - builder.append(" ").append(field.id).append(" -> {\n") - builder.append(" val localField = fieldsById[").append(field.id).append("]!!\n") - builder.append(" if (canReadRemoteField(remoteField, localField)) {\n") + builder.append(indent).append(" ").append(field.id).append(" -> {\n") + builder + .append(indent) + .append(" val localField = fieldsById[") + .append(field.id) + .append("]!!\n") + builder.append(indent).append(" if (canReadRemoteField(remoteField, localField)) {\n") + val readExpression = "readCompatibleFieldValue(readContext, remoteField, localField)" + val constructorReadExpression = + if (constructorRefs) "ctorFieldValue(readContext, $readExpression, type)" + else readExpression builder - .append(" ") + .append(indent) + .append(" ") .append(field.localName) .append(" = ") .append( castReadExpression( field, - "readCompatibleFieldValue(readContext, remoteField, localField)", + constructorReadExpression, compatible = true, ) ) .append("\n") - builder.append(" ") + builder.append(indent).append(" ") appendPresenceSet(field) builder.append("\n") - builder.append(" } else {\n") - builder.append(" skipField(readContext, remoteField)\n") - builder.append(" }\n") - builder.append(" }\n") - } - builder.append(" else -> skipField(readContext, remoteField)\n") - builder.append(" }\n") - builder.append(" }\n") - builder.append(" var missingDefaultMask = 0L\n") + builder.append(indent).append(" } else {\n") + builder.append(indent).append(" skipField(readContext, remoteField)\n") + builder.append(indent).append(" }\n") + builder.append(indent).append(" }\n") + } + builder.append(indent).append(" else -> skipField(readContext, remoteField)\n") + builder.append(indent).append(" }\n") + builder.append(indent).append("}\n") + builder.append(indent).append("var missingDefaultMask = 0L\n") val defaultFields = struct.fields.filter { it.hasDefault } for (field in struct.fields) { if (!field.hasDefault && field.nullable) { continue } - builder.append(" if (") + builder.append(indent).append("if (") appendPresenceMissing(field) builder.append(") {\n") when { field.hasDefault -> builder - .append(" missingDefaultMask = missingDefaultMask or ") + .append(indent) + .append(" missingDefaultMask = missingDefaultMask or ") .append(1L shl defaultFields.indexOf(field)) .append("L\n") else -> builder - .append(" throw DeserializationException(\"Required Kotlin field ") + .append(indent) + .append(" throw DeserializationException(\"Required Kotlin field ") .append(struct.qualifiedTypeName) .append('.') .append(field.name) .append(" is missing in compatible xlang payload\")\n") } - builder.append(" }\n") + builder.append(indent).append("}\n") } - writeDefaultDispatch() - builder.append(" }\n\n") + if (constructorRefs) { + builder.append(indent).append("checkNoUnresolvedReadRef(readContext)\n") + } + writeDefaultDispatch(indent, constructorRefs) } private fun writeMutableCompatibleReadBody() { @@ -713,11 +760,11 @@ internal class KotlinSerializerSourceWriter(private val struct: KotlinSourceStru builder.append(" return value\n") } - private fun writePresenceVars() { + private fun writePresenceVars(indent: String = " ") { val maxId = struct.fields.maxOfOrNull { it.id } ?: -1 val chunks = maxId / java.lang.Long.SIZE + 1 for (index in 0 until chunks) { - builder.append(" var presentMask").append(index).append(" = 0L\n") + builder.append(indent).append("var presentMask").append(index).append(" = 0L\n") } } @@ -853,10 +900,11 @@ internal class KotlinSerializerSourceWriter(private val struct: KotlinSourceStru ) } - private fun writeLocalDeclarations() { + private fun writeLocalDeclarations(indent: String = " ") { for (field in struct.fields) { builder - .append(" var ") + .append(indent) + .append("var ") .append(field.localName) .append(": ") .append(localVariableType(field)) @@ -866,15 +914,27 @@ internal class KotlinSerializerSourceWriter(private val struct: KotlinSourceStru } } - private fun writeDefaultDispatch() { + private fun writeDefaultDispatch(indent: String = " ", referenceConstructor: Boolean = false) { val defaultFields = struct.fields.filter { it.hasDefault } if (defaultFields.isEmpty()) { - builder.append(" return ") - appendConstructorCall(defaultMask = 0L) - builder.append("\n") + if (referenceConstructor) { + builder.append(indent).append("val constructed = ") + appendConstructorCall(defaultMask = 0L) + builder.append("\n") + builder.append(indent).append("referenceConstructorRef(readContext, constructed)\n") + builder.append(indent).append("return constructed\n") + } else { + builder.append(indent).append("return ") + appendConstructorCall(defaultMask = 0L) + builder.append("\n") + } return } - builder.append(" return when (missingDefaultMask) {\n") + if (referenceConstructor) { + builder.append(indent).append("val constructed = when (missingDefaultMask) {\n") + } else { + builder.append(indent).append("return when (missingDefaultMask) {\n") + } val combinations = 1L shl defaultFields.size for (combination in 0 until combinations) { var mask = 0L @@ -883,15 +943,20 @@ internal class KotlinSerializerSourceWriter(private val struct: KotlinSourceStru mask = mask or (1L shl i) } } - builder.append(" ").append(mask).append("L -> ") + builder.append(indent).append(" ").append(mask).append("L -> ") appendConstructorCall(defaultMask = mask) builder.append("\n") } builder.append( - " else -> throw DeserializationException(\"Unsupported Kotlin default argument mask \${missingDefaultMask} for " + indent + + " else -> throw DeserializationException(\"Unsupported Kotlin default argument mask \${missingDefaultMask} for " ) builder.append(struct.qualifiedTypeName).append("\")\n") - builder.append(" }\n") + builder.append(indent).append("}\n") + if (referenceConstructor) { + builder.append(indent).append("referenceConstructorRef(readContext, constructed)\n") + builder.append(indent).append("return constructed\n") + } } private fun appendConstructorCall(defaultMask: Long) { diff --git a/kotlin/fory-kotlin-ksp/src/test/kotlin/org/apache/fory/kotlin/ksp/ProcessorValidationTest.kt b/kotlin/fory-kotlin-ksp/src/test/kotlin/org/apache/fory/kotlin/ksp/ProcessorValidationTest.kt index 477b6eb04c..763738cb02 100644 --- a/kotlin/fory-kotlin-ksp/src/test/kotlin/org/apache/fory/kotlin/ksp/ProcessorValidationTest.kt +++ b/kotlin/fory-kotlin-ksp/src/test/kotlin/org/apache/fory/kotlin/ksp/ProcessorValidationTest.kt @@ -22,6 +22,7 @@ package org.apache.fory.kotlin.ksp import com.google.devtools.ksp.symbol.ClassKind import com.google.devtools.ksp.symbol.Modifier import org.testng.Assert.assertEquals +import org.testng.Assert.assertFalse import org.testng.Assert.assertNull import org.testng.Assert.assertTrue import org.testng.annotations.Test @@ -82,6 +83,10 @@ class ProcessorValidationTest { constructorBindingError(listOf("name"), listOf("name", "age"), "example.User"), "@ForyConstructor on example.User must declare one field name for each primary constructor parameter", ) + assertEquals( + constructorBindingError(emptyList(), emptyList(), "example.User"), + "@ForyConstructor on example.User must declare at least one field name", + ) assertEquals( constructorBindingError(listOf("name", "age"), listOf("name", "name"), "example.User"), "@ForyConstructor on example.User declares duplicate field name name", @@ -143,6 +148,63 @@ class ProcessorValidationTest { assertTrue(source.contains("objectCreator.newInstanceWithArguments(*constructorArgs")) } + @Test + fun defaultsUseGeneratedCompatibleRead() { + val stringType = + KotlinSourceTypeNode( + rawClassExpression = "String::class.java", + kotlinTypeName = "kotlin.String", + valueTypeName = "String", + typeName = "java.lang.String", + typeId = "Types.STRING", + nullable = false, + trackingRef = false, + primitive = false, + unsigned = false, + ) + val source = + KotlinSerializerSourceWriter( + KotlinSourceStruct( + packageName = "example", + typeName = "User", + qualifiedTypeName = "example.User", + serializerName = "User_ForySerializer", + serializerVisibility = KotlinSerializerVisibility.PUBLIC, + fields = + listOf( + KotlinSourceField( + id = 0, + name = "name", + type = stringType, + hasForyField = true, + foryFieldId = 1, + trackingRef = false, + dynamic = "AUTO", + arrayType = false, + hasDefault = true, + nullable = false, + propertyTypeName = "String", + constructorParameterName = "name", + ) + ), + originatingFiles = emptyList(), + ) + ) + .write() + val compatibleStart = source.indexOf("override fun readCompatible") + val copyStart = source.indexOf("override fun copy") + val compatibleSource = source.substring(compatibleStart, copyStart) + + assertFalse(compatibleSource.contains("return readCompatibleConstructor(readContext)")) + assertTrue(compatibleSource.contains("return readCompatibleDefaultConstructor(readContext)")) + assertTrue(compatibleSource.contains("beginConstructorRef(readContext)")) + assertTrue(compatibleSource.contains("checkNoUnresolvedReadRef(readContext)")) + assertTrue(compatibleSource.contains("referenceConstructorRef(readContext, constructed)")) + assertTrue(compatibleSource.contains("endConstructorRef(readContext)")) + assertTrue(compatibleSource.contains("missingDefaultMask")) + assertTrue(compatibleSource.contains("val constructed = when (missingDefaultMask)")) + } + @Test fun tracksWidePresence() { val intType = diff --git a/kotlin/fory-kotlin-tests/pom.xml b/kotlin/fory-kotlin-tests/pom.xml index f13af7b46f..35f12436b0 100644 --- a/kotlin/fory-kotlin-tests/pom.xml +++ b/kotlin/fory-kotlin-tests/pom.xml @@ -105,7 +105,6 @@ compile - true ${project.basedir}/src/main/kotlin ${project.build.directory}/generated-sources/ksp diff --git a/kotlin/fory-kotlin-tests/src/main/kotlin/org/apache/fory/kotlin/xlang/KotlinXlangPeer.kt b/kotlin/fory-kotlin-tests/src/main/kotlin/org/apache/fory/kotlin/xlang/KotlinXlangPeer.kt index 563c311f62..caaf24756a 100644 --- a/kotlin/fory-kotlin-tests/src/main/kotlin/org/apache/fory/kotlin/xlang/KotlinXlangPeer.kt +++ b/kotlin/fory-kotlin-tests/src/main/kotlin/org/apache/fory/kotlin/xlang/KotlinXlangPeer.kt @@ -31,6 +31,7 @@ import org.apache.fory.BaseFory import org.apache.fory.Fory import org.apache.fory.annotation.ArrayType import org.apache.fory.annotation.ForyCase +import org.apache.fory.annotation.ForyConstructor import org.apache.fory.annotation.ForyField import org.apache.fory.annotation.ForyStruct import org.apache.fory.annotation.ForyUnion @@ -50,26 +51,44 @@ import org.apache.fory.type.Types import org.apache.fory.type.union.UnknownCase @ForyStruct -public data class KotlinUser( +public data class KotlinUser +@ForyConstructor("id", "name", "score") +constructor( @ForyField(id = 1) val id: @Fixed UInt, @ForyField(id = 2) val name: String = "anonymous", @ForyField(id = 3) val score: @VarInt Long, ) @ForyStruct -public data class KotlinRegisteredSwap( +public data class KotlinRegisteredSwap +@ForyConstructor("left", "right") +constructor( @ForyField(id = 1) val left: String, @ForyField(id = 2) val right: String, ) @ForyStruct -internal data class KotlinInternalUser( +internal data class KotlinInternalUser +@ForyConstructor("id", "name") +constructor( @ForyField(id = 1) val id: UInt, @ForyField(id = 2) val name: String = "internal", ) @ForyStruct -public data class KotlinConcreteCollections( +public data class KotlinConcreteCollections +@ForyConstructor( + "names", + "values", + "tags", + "counts", + "mutableNames", + "mutableTags", + "mutableCounts", + "sortedNames", + "concurrentCounts", +) +constructor( @ForyField(id = 1) val names: ArrayList, @ForyField(id = 2) val values: java.util.LinkedList, @ForyField(id = 3) val tags: CopyOnWriteArrayList, @@ -82,7 +101,9 @@ public data class KotlinConcreteCollections( ) @ForyStruct -public data class KotlinUnsignedCollections( +public data class KotlinUnsignedCollections +@ForyConstructor("ids", "optionalIds", "totals", "byName", "namesById") +constructor( @ForyField(id = 1) val ids: List, @ForyField(id = 2) val optionalIds: List, @ForyField(id = 3) val totals: Set, @@ -91,7 +112,22 @@ public data class KotlinUnsignedCollections( ) @ForyStruct -public data class KotlinSchemaSurface( +public data class KotlinSchemaSurface +@ForyConstructor( + "nullableNames", + "dynamicList", + "dynamicValues", + "bytesAsArray", + "bits", + "unsignedLongs", + "fieldSiteId", + "denseIds", + "noRefUser", + "noRefUsers", + "chunks", + "chunksByName", +) +constructor( @ForyField(id = 1) val nullableNames: List?, @ForyField(id = 2) val dynamicList: List<*>, @ForyField(id = 3) val dynamicValues: Map, @@ -107,7 +143,22 @@ public data class KotlinSchemaSurface( ) @ForyStruct -public data class KotlinDenseArrays( +public data class KotlinDenseArrays +@ForyConstructor( + "ubytes", + "ushorts", + "uints", + "ulongs", + "ints", + "longs", + "bytes", + "shorts", + "floats", + "doubles", + "booleans", + "nullableUInts", +) +constructor( @ForyField(id = 1) val ubytes: UByteArray, @ForyField(id = 2) val ushorts: UShortArray, @ForyField(id = 3) val uints: UIntArray, @@ -122,10 +173,15 @@ public data class KotlinDenseArrays( @ForyField(id = 12) val nullableUInts: UIntArray?, ) -@ForyStruct public data class KotlinNullableCompatibleWriter(@ForyField(id = 1) val anchor: String) +@ForyStruct +public data class KotlinNullableCompatibleWriter +@ForyConstructor("anchor") +constructor(@ForyField(id = 1) val anchor: String) @ForyStruct -public data class KotlinNullableCompatibleReader( +public data class KotlinNullableCompatibleReader +@ForyConstructor("anchor", "maybeBoolean", "maybeInt", "maybeLong", "maybeUInt", "maybeULong") +constructor( @ForyField(id = 1) val anchor: String, @ForyField(id = 2) val maybeBoolean: Boolean?, @ForyField(id = 3) val maybeInt: Int?, @@ -135,7 +191,39 @@ public data class KotlinNullableCompatibleReader( ) @ForyStruct -public data class KotlinDurationAndHalfArrays( +public data class KotlinDefaultCompatibleWriter +@ForyConstructor("id") +constructor(@ForyField(id = 1) val id: Int) + +@ForyStruct +public data class KotlinDefaultCompatibleReader +@ForyConstructor("id", "name") +constructor( + @ForyField(id = 1) val id: Int, + @ForyField(id = 2) val name: String = "generated-default", +) + +@ForyStruct +public data class KotlinDefaultRefWriter +@ForyConstructor("id", "next") +constructor( + @ForyField(id = 1) val id: Int, + @Ref @ForyField(id = 2) var next: KotlinDefaultRefWriter?, +) + +@ForyStruct +public data class KotlinDefaultRefReader +@ForyConstructor("id", "name", "next") +constructor( + @ForyField(id = 1) val id: Int, + @ForyField(id = 3) val name: String = "generated-default", + @Ref @ForyField(id = 2) var next: KotlinDefaultRefReader?, +) + +@ForyStruct +public data class KotlinDurationAndHalfArrays +@ForyConstructor("duration", "float16s", "bfloat16s") +constructor( @ForyField(id = 1) val duration: kotlin.time.Duration, @ForyField(id = 2) val float16s: Float16Array, @ForyField(id = 3) val bfloat16s: BFloat16Array, @@ -148,6 +236,16 @@ public class KotlinMutableNode() { @Ref @ForyField(id = 2) public var parent: KotlinMutableNode? = null } +@ForyStruct +public class KotlinCtorBackrefRoot +@ForyConstructor("child") +constructor(@ForyField(id = 1) val child: KotlinCtorBackrefChild) + +@ForyStruct +public class KotlinCtorBackrefChild { + @ForyField(id = 1) public var root: KotlinCtorBackrefRoot? = null +} + @ForyUnion public sealed class KotlinPet { @ForyUnknownCase public data class Unknown(val value: UnknownCase) : KotlinPet() @@ -163,6 +261,8 @@ public fun main(args: Array) { } when (args[0]) { "static_serializer_round_trip" -> staticSerializerRoundTrip(args[1]) + "compatible_default_round_trip" -> compatibleDefaultRoundTrip() + "constructor_backref_copy" -> constructorBackrefCopy() "dense_array_round_trip" -> denseArrayRoundTrip(args[1]) "unsigned_collection_round_trip" -> unsignedCollectionRoundTrip(args[1]) else -> throw IllegalArgumentException("Unsupported Kotlin xlang case ${args[0]}") @@ -213,7 +313,9 @@ private fun staticSerializerRoundTrip(dataFile: String) { val swapped = KotlinRegisteredSwap(left = "right", right = "left") check(fory.deserialize(fory.serialize(swap), KotlinRegisteredSwap::class.java) == swapped) check(fory.copy(swap) == swapped) - check(fory.getSerializer(KotlinRegisteredSwap::class.java) is StaticGeneratedStructSerializer<*>) { + check( + fory.getSerializer(KotlinRegisteredSwap::class.java) is StaticGeneratedStructSerializer<*> + ) { "KotlinRegisteredSwap did not load a static generated serializer" } @@ -375,6 +477,8 @@ private fun staticSerializerRoundTrip(dataFile: String) { check(compatibleDecoded.maybeUInt == null) check(compatibleDecoded.maybeULong == null) + compatibleDefaultRoundTrip() + val durationAndHalfArrays = KotlinDurationAndHalfArrays( duration = (-500).milliseconds, @@ -427,6 +531,53 @@ private fun staticSerializerRoundTrip(dataFile: String) { } } +private fun constructorBackrefCopy() { + val refFory = + ForyKotlin.builder() + .withXlang(false) + .requireClassRegistration(true) + .withRefTracking(true) + .withRefCopy(true) + .build() + refFory.register() + refFory.register() + checkConstructorBackrefCopy(refFory) +} + +private fun checkConstructorBackrefCopy(fory: Fory) { + val child = KotlinCtorBackrefChild() + val root = KotlinCtorBackrefRoot(child) + child.root = root + try { + fory.copy(root) + error("Constructor back-reference was copied") + } catch (_: ForyException) {} +} + +private fun compatibleDefaultRoundTrip() { + val writer = newCompatibleFory() + writer.register("kotlin", "DefaultCompatible") + val reader = newCompatibleFory() + reader.register("kotlin", "DefaultCompatible") + val decoded = + reader.deserialize( + writer.serialize(KotlinDefaultCompatibleWriter(7)), + KotlinDefaultCompatibleReader::class.java, + ) + check(decoded == KotlinDefaultCompatibleReader(id = 7)) + + val refWriter = newRefCompatibleFory() + refWriter.register("kotlin", "DefaultRef") + val refReader = newRefCompatibleFory() + refReader.register("kotlin", "DefaultRef") + val node = KotlinDefaultRefWriter(id = 9, next = null) + node.next = node + try { + refReader.deserialize(refWriter.serialize(node), KotlinDefaultRefReader::class.java) + error("Constructor self-reference with a defaulted compatible field was deserialized") + } catch (_: ForyException) {} +} + private fun checkNoArgRegisterReceivers() { checkNoArgRegister(newFory()) checkNoArgRegister( @@ -524,6 +675,14 @@ private fun newCompatibleFory(): Fory = .withRefTracking(false) .build() +private fun newRefCompatibleFory(): Fory = + ForyKotlin.builder() + .withXlang(true) + .withCompatible(true) + .requireClassRegistration(true) + .withRefTracking(true) + .build() + private fun newRefFory(): Fory = ForyKotlin.builder() .withXlang(true) diff --git a/kotlin/fory-kotlin/pom.xml b/kotlin/fory-kotlin/pom.xml index ef5e09db65..d37985b21c 100644 --- a/kotlin/fory-kotlin/pom.xml +++ b/kotlin/fory-kotlin/pom.xml @@ -44,7 +44,6 @@ compile - true ${project.basedir}/src/main/kotlin ${project.basedir}/src/main/java @@ -61,7 +60,6 @@ test-compile - true ${project.basedir}/src/test/kotlin diff --git a/kotlin/fory-kotlin/src/main/java/org/apache/fory/serializer/kotlin/KotlinSerializers.java b/kotlin/fory-kotlin/src/main/java/org/apache/fory/serializer/kotlin/KotlinSerializers.java index c830124196..36acb0c5d8 100644 --- a/kotlin/fory-kotlin/src/main/java/org/apache/fory/serializer/kotlin/KotlinSerializers.java +++ b/kotlin/fory-kotlin/src/main/java/org/apache/fory/serializer/kotlin/KotlinSerializers.java @@ -125,38 +125,20 @@ public static void registerSerializers(Fory fory) { // Ranges and Progressions. registerIfAbsent(resolver, kotlin.ranges.CharRange.class); - resolver.registerSerializer(kotlin.ranges.CharRange.class, new CharRangeSerializer(config)); registerIfAbsent(resolver, kotlin.ranges.CharProgression.class); - resolver.registerSerializer( - kotlin.ranges.CharProgression.class, new CharProgressionSerializer(config)); registerIfAbsent(resolver, kotlin.ranges.IntRange.class); - resolver.registerSerializer(kotlin.ranges.IntRange.class, new IntRangeSerializer(config)); registerIfAbsent(resolver, kotlin.ranges.IntProgression.class); - resolver.registerSerializer( - kotlin.ranges.IntProgression.class, new IntProgressionSerializer(config)); registerIfAbsent(resolver, kotlin.ranges.LongRange.class); - resolver.registerSerializer(kotlin.ranges.LongRange.class, new LongRangeSerializer(config)); registerIfAbsent(resolver, kotlin.ranges.LongProgression.class); - resolver.registerSerializer( - kotlin.ranges.LongProgression.class, new LongProgressionSerializer(config)); registerIfAbsent(resolver, kotlin.ranges.UIntRange.class); - resolver.registerSerializer(kotlin.ranges.UIntRange.class, new UIntRangeSerializer(config)); registerIfAbsent(resolver, kotlin.ranges.UIntProgression.class); - resolver.registerSerializer( - kotlin.ranges.UIntProgression.class, new UIntProgressionSerializer(config)); registerIfAbsent(resolver, kotlin.ranges.ULongRange.class); - resolver.registerSerializer(kotlin.ranges.ULongRange.class, new ULongRangeSerializer(config)); registerIfAbsent(resolver, kotlin.ranges.ULongProgression.class); - resolver.registerSerializer( - kotlin.ranges.ULongProgression.class, new ULongProgressionSerializer(config)); // Built-in classes. registerIfAbsent(resolver, kotlin.Pair.class); - resolver.registerSerializer(kotlin.Pair.class, new PairSerializer(config)); registerIfAbsent(resolver, kotlin.Triple.class); - resolver.registerSerializer(kotlin.Triple.class, new TripleSerializer(config)); registerIfAbsent(resolver, kotlin.Result.class); - resolver.registerSerializer(kotlin.Result.class, new ResultSerializer(config)); registerIfAbsent(resolver, Result.Failure.class); // kotlin.random diff --git a/kotlin/fory-kotlin/src/main/kotlin/org/apache/fory/serializer/kotlin/KotlinBuiltinSerializers.kt b/kotlin/fory-kotlin/src/main/kotlin/org/apache/fory/serializer/kotlin/KotlinBuiltinSerializers.kt deleted file mode 100644 index 56362b8525..0000000000 --- a/kotlin/fory-kotlin/src/main/kotlin/org/apache/fory/serializer/kotlin/KotlinBuiltinSerializers.kt +++ /dev/null @@ -1,236 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.fory.serializer.kotlin - -import org.apache.fory.config.Config -import org.apache.fory.context.ReadContext -import org.apache.fory.context.WriteContext -import org.apache.fory.serializer.ImmutableSerializer -import org.apache.fory.serializer.Shareable - -public class PairSerializer(config: Config) : - ImmutableSerializer>(config, Pair::class.java), Shareable { - override fun write(writeContext: WriteContext, value: Pair<*, *>) { - writeContext.writeRef(value.first) - writeContext.writeRef(value.second) - } - - override fun read(readContext: ReadContext): Pair<*, *> { - return Pair(readContext.readRef(), readContext.readRef()) - } -} - -public class TripleSerializer(config: Config) : - ImmutableSerializer>(config, Triple::class.java), Shareable { - override fun write(writeContext: WriteContext, value: Triple<*, *, *>) { - writeContext.writeRef(value.first) - writeContext.writeRef(value.second) - writeContext.writeRef(value.third) - } - - override fun read(readContext: ReadContext): Triple<*, *, *> { - return Triple(readContext.readRef(), readContext.readRef(), readContext.readRef()) - } -} - -public class ResultSerializer(config: Config) : - ImmutableSerializer>(config, Result::class.java), Shareable { - override fun write(writeContext: WriteContext, value: Result<*>) { - val failure = value.exceptionOrNull() - val buffer = writeContext.buffer - buffer.writeBoolean(failure == null) - if (failure == null) { - writeContext.writeRef(value.getOrNull()) - } else { - writeContext.writeRef(failure) - } - } - - override fun read(readContext: ReadContext): Result<*> { - return if (readContext.buffer.readBoolean()) { - Result.success(readContext.readRef()) - } else { - Result.failure(readContext.readRef() as Throwable) - } - } -} - -public class CharRangeSerializer(config: Config) : - ImmutableSerializer(config, CharRange::class.java), Shareable { - override fun write(writeContext: WriteContext, value: CharRange) { - val buffer = writeContext.buffer - buffer.writeInt16(value.first.code.toShort()) - buffer.writeInt16(value.last.code.toShort()) - } - - override fun read(readContext: ReadContext): CharRange { - val buffer = readContext.buffer - return CharRange(buffer.readInt16().toInt().toChar(), buffer.readInt16().toInt().toChar()) - } -} - -public class CharProgressionSerializer(config: Config) : - ImmutableSerializer(config, CharProgression::class.java), Shareable { - override fun write(writeContext: WriteContext, value: CharProgression) { - val buffer = writeContext.buffer - buffer.writeInt16(value.first.code.toShort()) - buffer.writeInt16(value.last.code.toShort()) - buffer.writeInt32(value.step) - } - - override fun read(readContext: ReadContext): CharProgression { - val buffer = readContext.buffer - return CharProgression.fromClosedRange( - buffer.readInt16().toInt().toChar(), - buffer.readInt16().toInt().toChar(), - buffer.readInt32() - ) - } -} - -public class IntRangeSerializer(config: Config) : - ImmutableSerializer(config, IntRange::class.java), Shareable { - override fun write(writeContext: WriteContext, value: IntRange) { - val buffer = writeContext.buffer - buffer.writeInt32(value.first) - buffer.writeInt32(value.last) - } - - override fun read(readContext: ReadContext): IntRange { - val buffer = readContext.buffer - return IntRange(buffer.readInt32(), buffer.readInt32()) - } -} - -public class IntProgressionSerializer(config: Config) : - ImmutableSerializer(config, IntProgression::class.java), Shareable { - override fun write(writeContext: WriteContext, value: IntProgression) { - val buffer = writeContext.buffer - buffer.writeInt32(value.first) - buffer.writeInt32(value.last) - buffer.writeInt32(value.step) - } - - override fun read(readContext: ReadContext): IntProgression { - val buffer = readContext.buffer - return IntProgression.fromClosedRange( - buffer.readInt32(), - buffer.readInt32(), - buffer.readInt32() - ) - } -} - -public class LongRangeSerializer(config: Config) : - ImmutableSerializer(config, LongRange::class.java), Shareable { - override fun write(writeContext: WriteContext, value: LongRange) { - val buffer = writeContext.buffer - buffer.writeInt64(value.first) - buffer.writeInt64(value.last) - } - - override fun read(readContext: ReadContext): LongRange { - val buffer = readContext.buffer - return LongRange(buffer.readInt64(), buffer.readInt64()) - } -} - -public class LongProgressionSerializer(config: Config) : - ImmutableSerializer(config, LongProgression::class.java), Shareable { - override fun write(writeContext: WriteContext, value: LongProgression) { - val buffer = writeContext.buffer - buffer.writeInt64(value.first) - buffer.writeInt64(value.last) - buffer.writeInt64(value.step) - } - - override fun read(readContext: ReadContext): LongProgression { - val buffer = readContext.buffer - val first = buffer.readInt64() - val last = buffer.readInt64() - val step = buffer.readInt64() - return LongProgression.fromClosedRange(first, last, step) - } -} - -public class UIntRangeSerializer(config: Config) : - ImmutableSerializer(config, UIntRange::class.java), Shareable { - override fun write(writeContext: WriteContext, value: UIntRange) { - val buffer = writeContext.buffer - buffer.writeInt32(value.first.toInt()) - buffer.writeInt32(value.last.toInt()) - } - - override fun read(readContext: ReadContext): UIntRange { - val buffer = readContext.buffer - return UIntRange(buffer.readInt32().toUInt(), buffer.readInt32().toUInt()) - } -} - -public class UIntProgressionSerializer(config: Config) : - ImmutableSerializer(config, UIntProgression::class.java), Shareable { - override fun write(writeContext: WriteContext, value: UIntProgression) { - val buffer = writeContext.buffer - buffer.writeInt32(value.first.toInt()) - buffer.writeInt32(value.last.toInt()) - buffer.writeInt32(value.step) - } - - override fun read(readContext: ReadContext): UIntProgression { - val buffer = readContext.buffer - return UIntProgression.fromClosedRange( - buffer.readInt32().toUInt(), - buffer.readInt32().toUInt(), - buffer.readInt32() - ) - } -} - -public class ULongRangeSerializer(config: Config) : - ImmutableSerializer(config, ULongRange::class.java), Shareable { - override fun write(writeContext: WriteContext, value: ULongRange) { - val buffer = writeContext.buffer - buffer.writeInt64(value.first.toLong()) - buffer.writeInt64(value.last.toLong()) - } - - override fun read(readContext: ReadContext): ULongRange { - val buffer = readContext.buffer - return ULongRange(buffer.readInt64().toULong(), buffer.readInt64().toULong()) - } -} - -public class ULongProgressionSerializer(config: Config) : - ImmutableSerializer(config, ULongProgression::class.java), Shareable { - override fun write(writeContext: WriteContext, value: ULongProgression) { - val buffer = writeContext.buffer - buffer.writeInt64(value.first.toLong()) - buffer.writeInt64(value.last.toLong()) - buffer.writeInt64(value.step) - } - - override fun read(readContext: ReadContext): ULongProgression { - val buffer = readContext.buffer - val first = buffer.readInt64().toULong() - val last = buffer.readInt64().toULong() - val step = buffer.readInt64() - return ULongProgression.fromClosedRange(first, last, step) - } -} From 7cee7e1528ab2ee895d1061291d9c425c1d64934 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Tue, 2 Jun 2026 10:25:49 +0800 Subject: [PATCH 39/69] fix: stabilize jdk25 unsafe verification --- .agents/languages/java.md | 3 + integration_tests/graalvm_tests/README.md | 5 +- integration_tests/graalvm_tests/pom.xml | 3 - .../apache/fory/reflect/ObjectCreators.java | 20 ++-- .../serializer/AbstractObjectSerializer.java | 12 +-- .../fory/serializer/StringSerializer.java | 14 ++- .../fory-core/native-image.properties | 3 + .../ksp/KotlinSerializerSourceWriter.kt | 40 +++++--- .../kotlin/ksp/ProcessorValidationTest.kt | 91 +++++++++++++++++++ 9 files changed, 149 insertions(+), 42 deletions(-) diff --git a/.agents/languages/java.md b/.agents/languages/java.md index 9c8e2f30ce..310c78784f 100644 --- a/.agents/languages/java.md +++ b/.agents/languages/java.md @@ -22,6 +22,9 @@ Load this file when changing anything under `java/` or when Java drives a cross- - Do not add normal-JVM process-global caches keyed by user classes, generated classes, serializer classes, classloaders, or class-bound method handles. Prefer per-runtime state, immutable shared metadata, or build-time-only template data. - Concrete serializers may opt into sharing only after auditing retained fields. Treat serializers retaining `TypeResolver`, `RefResolver`, mutable scratch buffers, runtime state, or classloader-sensitive state as non-shareable unless that state is externalized. - Resolver and serializer hot paths should keep the fast-path/null-slow-path shape obvious. Hoist repeated buffer or cache-state access into locals for multi-step operations and keep rebuild/restoration logic cold. +- Hot-path feature gates that are runtime constants must be `static final` fields read directly in + the branch. Do not hide them behind helper methods such as `jdkInternalFieldAccess()`, because + that obscures branch folding and can leave avoidable call/inlining work in hot serializers. - In Java codec hot paths, avoid `Preconditions.checkArgument` for attacker-controlled primitive validation. Use direct primitive branches and throw on the cold error path to preserve inlining and avoid varargs/helper overhead. diff --git a/integration_tests/graalvm_tests/README.md b/integration_tests/graalvm_tests/README.md index 8073b9ee35..10473f941c 100644 --- a/integration_tests/graalvm_tests/README.md +++ b/integration_tests/graalvm_tests/README.md @@ -11,8 +11,5 @@ mvn -DskipTests=true -Pnative package ## Benchmark ```bash -BENCHMARK_REPEAT=400000 mvn -Pnative -Dagent=true -DskipTests -DskipNativeBuild=true package exec:exec@java-agent -BENCHMARK_REPEAT=400000 mvn -DskipTests=true -Pnative -Dagent=true package +BENCHMARK_REPEAT=400000 mvn -DskipTests=true -Pnative package ``` - -`-Dagent=true` is needed by JDK serialization only to build reflection config, it's not needed for fory serialization. diff --git a/integration_tests/graalvm_tests/pom.xml b/integration_tests/graalvm_tests/pom.xml index f2f9945dd8..dc211053db 100644 --- a/integration_tests/graalvm_tests/pom.xml +++ b/integration_tests/graalvm_tests/pom.xml @@ -169,9 +169,6 @@ false - - true - diff --git a/java/fory-core/src/main/java/org/apache/fory/reflect/ObjectCreators.java b/java/fory-core/src/main/java/org/apache/fory/reflect/ObjectCreators.java index 7784b3670f..dd361c1c86 100644 --- a/java/fory-core/src/main/java/org/apache/fory/reflect/ObjectCreators.java +++ b/java/fory-core/src/main/java/org/apache/fory/reflect/ObjectCreators.java @@ -203,16 +203,6 @@ private static ConstructorMatch explicitConstructor( type, (Constructor) annotatedConstructor, annotation.value(), "@ForyConstructor"); } - private static boolean isCompilerGeneratedConstructor(Constructor constructor) { - if (constructor.isSynthetic()) { - return true; - } - Class[] parameterTypes = constructor.getParameterTypes(); - return parameterTypes.length > 0 - && "kotlin.jvm.internal.DefaultConstructorMarker" - .equals(parameterTypes[parameterTypes.length - 1].getName()); - } - static ConstructorMatch explicitConstructor( Class type, Constructor constructor, String[] fieldNames, String source) { if (isJavaPlatformType(type)) { @@ -257,6 +247,16 @@ static ConstructorMatch explicitConstructor( return match; } + private static boolean isCompilerGeneratedConstructor(Constructor constructor) { + if (constructor.isSynthetic()) { + return true; + } + Class[] parameterTypes = constructor.getParameterTypes(); + return parameterTypes.length > 0 + && "kotlin.jvm.internal.DefaultConstructorMarker" + .equals(parameterTypes[parameterTypes.length - 1].getName()); + } + private static boolean isJavaPlatformType(Class type) { return type.getName().startsWith("java."); } diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/AbstractObjectSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/AbstractObjectSerializer.java index da195c6c38..87a8984f4f 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/AbstractObjectSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/AbstractObjectSerializer.java @@ -1303,6 +1303,12 @@ protected final void checkNoUnresolvedReadRef(ReadContext readContext) { checkNoUnresolvedReadRef(readContext, type); } + public static void checkNoUnresolvedReadRef(ReadContext readContext, Class type) { + if (consumeSelfRef(readContext)) { + throwConstructorCycle(type); + } + } + protected final Object beginConstructorCopy(CopyContext copyContext, Object originObj) { if (!copyContext.copyTrackingRef()) { return null; @@ -1416,12 +1422,6 @@ private static boolean isJdkClass(Class fieldClass) { || name.startsWith("sun."); } - public static void checkNoUnresolvedReadRef(ReadContext readContext, Class type) { - if (consumeSelfRef(readContext)) { - throwConstructorCycle(type); - } - } - public static void beginConstructorRef(ReadContext readContext) { if (readContext.hasPreservedRefId()) { constructorRefIds(readContext).add(readContext.lastPreservedRefId()); diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/StringSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/StringSerializer.java index a5095ab0c9..8a2d994ccb 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/StringSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/StringSerializer.java @@ -53,6 +53,8 @@ public final class StringSerializer extends ImmutableSerializer { PlatformStringUtils.STRING_VALUE_FIELD_IS_CHARS; private static final boolean STRING_VALUE_FIELD_IS_BYTES = PlatformStringUtils.STRING_VALUE_FIELD_IS_BYTES; + private static final boolean JDK_INTERNAL_FIELD_ACCESS = + PlatformStringUtils.JDK_STRING_FIELD_ACCESS; private static final byte LATIN1 = 0; private static final byte UTF16 = 1; @@ -62,10 +64,6 @@ public final class StringSerializer extends ImmutableSerializer { private static final boolean STRING_HAS_COUNT_OFFSET = PlatformStringUtils.STRING_HAS_COUNT_OFFSET; - private static boolean jdkInternalFieldAccess() { - return PlatformStringUtils.JDK_STRING_FIELD_ACCESS; - } - private final boolean compressString; private final boolean writeNumUtf16BytesForUtf8Encoding; private final boolean xlang; @@ -103,7 +101,7 @@ public String read(ReadContext readContext) { public static Expression writeStringExpr( Expression strSerializer, Expression buffer, Expression str, boolean compressString) { - if (!jdkInternalFieldAccess()) { + if (!JDK_INTERNAL_FIELD_ACCESS) { return new Invoke(strSerializer, "writeString", buffer, str); } if (STRING_VALUE_FIELD_IS_BYTES) { @@ -134,7 +132,7 @@ public static Expression writeStringExpr( public static Expression readStringExpr( Expression strSerializer, Expression buffer, boolean compressString) { - if (!jdkInternalFieldAccess()) { + if (!JDK_INTERNAL_FIELD_ACCESS) { return new Invoke(strSerializer, "readString", STRING_TYPE, buffer); } if (STRING_VALUE_FIELD_IS_BYTES) { @@ -301,7 +299,7 @@ public String readCompressedCharsString(MemoryBuffer buffer) { // Invoked by fory JIT public void writeString(MemoryBuffer buffer, String value) { - if (!jdkInternalFieldAccess()) { + if (!JDK_INTERNAL_FIELD_ACCESS) { writeStringSlow(buffer, value); return; } @@ -335,7 +333,7 @@ private void writeJava8String(MemoryBuffer buffer, String value) { // Invoked by fory JIT public String readString(MemoryBuffer buffer) { - if (!jdkInternalFieldAccess()) { + if (!JDK_INTERNAL_FIELD_ACCESS) { return readStringSlow(buffer); } if (STRING_VALUE_FIELD_IS_BYTES) { diff --git a/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties b/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties index 9f01c6e257..e6e32e0a65 100644 --- a/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties +++ b/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties @@ -161,6 +161,7 @@ Args=--initialize-at-build-time=org.apache.fory.annotation.ForyField$Dynamic,\ org.apache.fory.Fory,\ org.apache.fory.ThreadLocalFory,\ org.apache.fory.builder.AccessorHelper,\ + org.apache.fory.builder.UnsafeCodegenSupport,\ org.apache.fory.builder.JITContext,\ org.apache.fory.builder.ObjectCodecBuilder,\ org.apache.fory.builder.CompatibleCodecBuilder,\ @@ -244,6 +245,8 @@ Args=--initialize-at-build-time=org.apache.fory.annotation.ForyField$Dynamic,\ org.apache.fory.meta.TypeEqualMetaCompressor,\ org.apache.fory.pool.ThreadPoolFory,\ org.apache.fory.reflect.FieldAccessor$GeneratedAccessor,\ + org.apache.fory.reflect.FieldAccessorStrategy,\ + org.apache.fory.reflect.FieldAccessorStrategy$GeneratedAccessor,\ org.apache.fory.reflect.ReflectionUtils$1,\ org.apache.fory.reflect.ReflectionUtils$2,\ org.apache.fory.reflect.ReflectionUtils,\ diff --git a/kotlin/fory-kotlin-ksp/src/main/kotlin/org/apache/fory/kotlin/ksp/KotlinSerializerSourceWriter.kt b/kotlin/fory-kotlin-ksp/src/main/kotlin/org/apache/fory/kotlin/ksp/KotlinSerializerSourceWriter.kt index 995a786554..3302fa4259 100644 --- a/kotlin/fory-kotlin-ksp/src/main/kotlin/org/apache/fory/kotlin/ksp/KotlinSerializerSourceWriter.kt +++ b/kotlin/fory-kotlin-ksp/src/main/kotlin/org/apache/fory/kotlin/ksp/KotlinSerializerSourceWriter.kt @@ -334,12 +334,13 @@ internal class KotlinSerializerSourceWriter(private val struct: KotlinSourceStru ) builder.append(" return when (fieldId) {\n") for (field in struct.fields) { - val expression = + val readExpression = castReadExpression( field, "readCompatibleFieldValue(readContext, remoteField, localField)", compatible = true, ) + val expression = constructorReadExpression(field, readExpression) builder.append(" ").append(field.id).append(" -> ").append(expression).append("\n") } builder.append( @@ -399,12 +400,9 @@ internal class KotlinSerializerSourceWriter(private val struct: KotlinSourceStru builder .append(" fieldValues[") .append(field.id) - .append("] = copyConstructorFieldValue(copyContext, value, ") - .append("value.") - .append(field.name) - .append(", fieldsById[") - .append(field.id) - .append("]!!)\n") + .append("] = ") + .append(constructorCopyExpression(field)) + .append("\n") builder.append(" }\n") } builder.append( @@ -548,7 +546,9 @@ internal class KotlinSerializerSourceWriter(private val struct: KotlinSourceStru builder.append(" return when (fieldId) {\n") for (field in struct.fields) { val direct = directReadExpression(field) - val expression = direct ?: castReadExpression(field, "readFieldValue(readContext, fieldInfo)") + val readExpression = + direct ?: castReadExpression(field, "readFieldValue(readContext, fieldInfo)") + val expression = constructorReadExpression(field, readExpression) builder.append(" ").append(field.id).append(" -> ").append(expression).append("\n") } builder.append( @@ -900,6 +900,20 @@ internal class KotlinSerializerSourceWriter(private val struct: KotlinSourceStru ) } + private fun constructorCopyExpression(field: KotlinSourceField): String { + val expression = + when { + field.type.primitive || isScalarUnsigned(field) -> "value.${field.name}" + isDenseUnsignedArray(field) -> + if (field.nullable) "value.${field.name}?.copyOf()" else "value.${field.name}.copyOf()" + field.type.isCollectionOrMap() -> + copyContainerExpression(field.type, "value.${field.name}", 0) + else -> + "copyConstructorFieldValue(copyContext, value, value.${field.name}, fieldsById[${field.id}]!!)" + } + return constructorReadExpression(field, expression) + } + private fun writeLocalDeclarations(indent: String = " ") { for (field in struct.fields) { builder @@ -989,12 +1003,16 @@ internal class KotlinSerializerSourceWriter(private val struct: KotlinSourceStru } else { "${field.localName}!!" } + return constructorReadExpression(field, localValue) + } + + private fun constructorReadExpression(field: KotlinSourceField, valueExpression: String): String { if (field.type.collectionFactory == CollectionFactory.NONE) { - return localValue + return valueExpression } val conversion = collectionConversionFunction(field.type.collectionFactory) - return if (field.nullable) "$localValue?.let { $conversion(it) }" - else "$conversion($localValue)" + return if (field.nullable) "($valueExpression)?.let { $conversion(it) }" + else "$conversion($valueExpression)" } private fun localVariableType(field: KotlinSourceField): String { diff --git a/kotlin/fory-kotlin-ksp/src/test/kotlin/org/apache/fory/kotlin/ksp/ProcessorValidationTest.kt b/kotlin/fory-kotlin-ksp/src/test/kotlin/org/apache/fory/kotlin/ksp/ProcessorValidationTest.kt index 763738cb02..ab2e56df1d 100644 --- a/kotlin/fory-kotlin-ksp/src/test/kotlin/org/apache/fory/kotlin/ksp/ProcessorValidationTest.kt +++ b/kotlin/fory-kotlin-ksp/src/test/kotlin/org/apache/fory/kotlin/ksp/ProcessorValidationTest.kt @@ -148,6 +148,97 @@ class ProcessorValidationTest { assertTrue(source.contains("objectCreator.newInstanceWithArguments(*constructorArgs")) } + @Test + fun constructorFieldsAdaptCollections() { + val stringType = + KotlinSourceTypeNode( + rawClassExpression = "String::class.java", + kotlinTypeName = "kotlin.String", + valueTypeName = "String", + typeName = "java.lang.String", + typeId = "Types.STRING", + nullable = false, + trackingRef = false, + primitive = false, + unsigned = false, + ) + val intType = + KotlinSourceTypeNode( + rawClassExpression = "Int::class.javaPrimitiveType!!", + kotlinTypeName = "kotlin.Int", + valueTypeName = "Int", + typeName = "int32", + typeId = "Types.VARINT32", + nullable = false, + trackingRef = false, + primitive = true, + unsigned = false, + ) + val mapType = + KotlinSourceTypeNode( + rawClassExpression = "java.util.Map::class.java", + kotlinTypeName = "java.util.TreeMap", + valueTypeName = "kotlin.collections.Map", + typeName = "java.util.Map", + typeId = "Types.MAP", + nullable = false, + trackingRef = false, + primitive = false, + unsigned = false, + collectionFactory = CollectionFactory.TREE_MAP, + typeArguments = listOf(stringType, intType), + ) + val source = + KotlinSerializerSourceWriter( + KotlinSourceStruct( + packageName = "example", + typeName = "User", + qualifiedTypeName = "example.User", + serializerName = "User_ForySerializer", + serializerVisibility = KotlinSerializerVisibility.PUBLIC, + fields = + listOf( + KotlinSourceField( + id = 0, + name = "counts", + type = mapType, + hasForyField = true, + foryFieldId = 1, + trackingRef = false, + dynamic = "AUTO", + arrayType = false, + hasDefault = false, + nullable = false, + propertyTypeName = "java.util.TreeMap", + constructorParameterName = "counts", + ) + ), + originatingFiles = emptyList(), + ) + ) + .write() + + assertTrue( + source.contains("return User(counts = KotlinCollectionAdapters.toTreeMap(field0!!))") + ) + assertTrue( + source.contains( + "0 -> KotlinCollectionAdapters.toTreeMap((readFieldValue(readContext, fieldInfo) as kotlin.collections.Map))" + ) + ) + assertTrue( + source.contains( + "0 -> KotlinCollectionAdapters.toTreeMap((readCompatibleFieldValue(readContext, remoteField, localField) as kotlin.collections.Map))" + ) + ) + assertTrue( + source.contains( + "fieldValues[0] = KotlinCollectionAdapters.toTreeMap(run { val copySource0 = value.counts; val copyTarget0 = java.util.LinkedHashMap(copySource0.size);" + ) + ) + assertFalse(source.contains("fieldValues[0] = copyConstructorFieldValue")) + } + @Test fun defaultsUseGeneratedCompatibleRead() { val stringType = From e14b8672cabb47309c365f8a9ab8a7cba1ccbb73 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Tue, 2 Jun 2026 11:55:19 +0800 Subject: [PATCH 40/69] fix: preserve kotlin constructor containers --- .../fory-core/native-image.properties | 1 - .../ksp/KotlinSerializerSourceWriter.kt | 174 +++++++++++++++--- .../kotlin/ksp/ProcessorValidationTest.kt | 105 ++++++++++- .../fory/kotlin/xlang/KotlinXlangPeer.kt | 22 +++ 4 files changed, 274 insertions(+), 28 deletions(-) diff --git a/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties b/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties index e6e32e0a65..71837d675c 100644 --- a/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties +++ b/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties @@ -244,7 +244,6 @@ Args=--initialize-at-build-time=org.apache.fory.annotation.ForyField$Dynamic,\ org.apache.fory.meta.MetaStringEncoder,\ org.apache.fory.meta.TypeEqualMetaCompressor,\ org.apache.fory.pool.ThreadPoolFory,\ - org.apache.fory.reflect.FieldAccessor$GeneratedAccessor,\ org.apache.fory.reflect.FieldAccessorStrategy,\ org.apache.fory.reflect.FieldAccessorStrategy$GeneratedAccessor,\ org.apache.fory.reflect.ReflectionUtils$1,\ diff --git a/kotlin/fory-kotlin-ksp/src/main/kotlin/org/apache/fory/kotlin/ksp/KotlinSerializerSourceWriter.kt b/kotlin/fory-kotlin-ksp/src/main/kotlin/org/apache/fory/kotlin/ksp/KotlinSerializerSourceWriter.kt index 3302fa4259..93cf8a06da 100644 --- a/kotlin/fory-kotlin-ksp/src/main/kotlin/org/apache/fory/kotlin/ksp/KotlinSerializerSourceWriter.kt +++ b/kotlin/fory-kotlin-ksp/src/main/kotlin/org/apache/fory/kotlin/ksp/KotlinSerializerSourceWriter.kt @@ -907,7 +907,7 @@ internal class KotlinSerializerSourceWriter(private val struct: KotlinSourceStru isDenseUnsignedArray(field) -> if (field.nullable) "value.${field.name}?.copyOf()" else "value.${field.name}.copyOf()" field.type.isCollectionOrMap() -> - copyContainerExpression(field.type, "value.${field.name}", 0) + return copyContainerExpression(field.type, "value.${field.name}", 0) else -> "copyConstructorFieldValue(copyContext, value, value.${field.name}, fieldsById[${field.id}]!!)" } @@ -1007,12 +1007,106 @@ internal class KotlinSerializerSourceWriter(private val struct: KotlinSourceStru } private fun constructorReadExpression(field: KotlinSourceField, valueExpression: String): String { - if (field.type.collectionFactory == CollectionFactory.NONE) { + return collectionReadExpression(field.type, valueExpression, 0, erasedInput = false) + } + + private fun collectionReadExpression( + type: KotlinSourceTypeNode, + valueExpression: String, + depth: Int, + erasedInput: Boolean + ): String { + if (!type.isCollectionOrMap()) { return valueExpression } - val conversion = collectionConversionFunction(field.type.collectionFactory) - return if (field.nullable) "($valueExpression)?.let { $conversion(it) }" - else "$conversion($valueExpression)" + if (type.nullable) { + val value = "readValue$depth" + return "$valueExpression?.let { $value -> ${collectionReadExpression(type.copy(nullable = false), value, depth + 1, erasedInput)} }" + } + val adapted = + if (type.typeArguments.any { needsCollectionReadAdaptation(it) }) { + readContainerExpression(type, valueExpression, depth) + } else { + valueExpression + } + return applyCollectionFactory(type, adapted, erasedInput && adapted == valueExpression) + } + + private fun needsCollectionReadAdaptation(type: KotlinSourceTypeNode): Boolean = + type.collectionFactory != CollectionFactory.NONE || + (type.isCollectionOrMap() && type.typeArguments.any { needsCollectionReadAdaptation(it) }) + + private fun readContainerExpression( + type: KotlinSourceTypeNode, + expression: String, + depth: Int + ): String { + val source = "readSource$depth" + val target = "readTarget$depth" + val valueType = type.valueTypeName.removeSuffix("?") + return when (type.typeId) { + "Types.LIST" -> { + val element = "readElement$depth" + val adaptedElement = readElementExpression(type.typeArguments[0], element, depth + 1) + "run { val $source = ${erasedCollectionExpression(type, expression)}; val $target = java.util.ArrayList($source.size); for ($element in $source) { $target.add($adaptedElement) }; $target as $valueType }" + } + "Types.SET" -> { + val element = "readElement$depth" + val adaptedElement = readElementExpression(type.typeArguments[0], element, depth + 1) + "run { val $source = ${erasedCollectionExpression(type, expression)}; val $target = java.util.LinkedHashSet($source.size); for ($element in $source) { $target.add($adaptedElement) }; $target as $valueType }" + } + "Types.MAP" -> { + val entry = "readEntry$depth" + val adaptedKey = readElementExpression(type.typeArguments[0], "$entry.key", depth + 1) + val adaptedValue = readElementExpression(type.typeArguments[1], "$entry.value", depth + 2) + "run { val $source = ${erasedCollectionExpression(type, expression)}; val $target = java.util.LinkedHashMap($source.size); for ($entry in $source.entries) { $target[$adaptedKey] = $adaptedValue }; $target as $valueType }" + } + else -> expression + } + } + + private fun readElementExpression( + type: KotlinSourceTypeNode, + expression: String, + depth: Int + ): String { + if (!needsCollectionReadAdaptation(type)) { + return expression + } + return collectionReadExpression(type, expression, depth, erasedInput = true) + } + + private fun erasedCollectionExpression(type: KotlinSourceTypeNode, expression: String): String = + when (type.typeId) { + "Types.LIST", + "Types.SET" -> "($expression as Collection<*>)" + "Types.MAP" -> "($expression as Map<*, *>)" + else -> expression + } + + private fun typedCollectionExpression(type: KotlinSourceTypeNode, expression: String): String { + if (!type.isCollectionOrMap()) { + return expression + } + return "(${erasedCollectionExpression(type, expression)} as ${type.valueTypeName.removeSuffix("?")})" + } + + private fun applyCollectionFactory( + type: KotlinSourceTypeNode, + valueExpression: String, + erasedInput: Boolean = false + ): String { + if (type.collectionFactory == CollectionFactory.NONE) { + return valueExpression + } + val conversion = collectionConversionFunction(type.collectionFactory) + if (!erasedInput) { + return "$conversion($valueExpression)" + } + return typedCollectionExpression( + type, + "$conversion(${erasedCollectionExpression(type, valueExpression)})" + ) } private fun localVariableType(field: KotlinSourceField): String { @@ -1095,25 +1189,27 @@ internal class KotlinSerializerSourceWriter(private val struct: KotlinSourceStru val source = "copySource$depth" val target = "copyTarget$depth" val valueType = type.valueTypeName.removeSuffix("?") - return when (type.typeId) { - "Types.LIST" -> { - val element = "copyElement$depth" - val copiedElement = copyElementExpression(type.typeArguments[0], element, depth + 1) - "run { val $source = $expression; val $target = java.util.ArrayList($source.size); for ($element in $source) { $target.add($copiedElement) }; $target as $valueType }" - } - "Types.SET" -> { - val element = "copyElement$depth" - val copiedElement = copyElementExpression(type.typeArguments[0], element, depth + 1) - "run { val $source = $expression; val $target = java.util.LinkedHashSet($source.size); for ($element in $source) { $target.add($copiedElement) }; $target as $valueType }" - } - "Types.MAP" -> { - val entry = "copyEntry$depth" - val copiedKey = copyElementExpression(type.typeArguments[0], "$entry.key", depth + 1) - val copiedValue = copyElementExpression(type.typeArguments[1], "$entry.value", depth + 2) - "run { val $source = $expression; val $target = java.util.LinkedHashMap($source.size); for ($entry in $source.entries) { $target[$copiedKey] = $copiedValue }; $target as $valueType }" + val copied = + when (type.typeId) { + "Types.LIST" -> { + val element = "copyElement$depth" + val copiedElement = copyElementExpression(type.typeArguments[0], element, depth + 1) + "run { val $source = $expression; val $target = java.util.ArrayList($source.size); for ($element in $source) { $target.add($copiedElement) }; $target as $valueType }" + } + "Types.SET" -> { + val element = "copyElement$depth" + val copiedElement = copyElementExpression(type.typeArguments[0], element, depth + 1) + "run { val $source = $expression; val $target = java.util.LinkedHashSet($source.size); for ($element in $source) { $target.add($copiedElement) }; $target as $valueType }" + } + "Types.MAP" -> { + val entry = "copyEntry$depth" + val copiedKey = copyElementExpression(type.typeArguments[0], "$entry.key", depth + 1) + val copiedValue = copyElementExpression(type.typeArguments[1], "$entry.value", depth + 2) + "run { val $source = $expression; val $target = java.util.LinkedHashMap($source.size); for ($entry in $source.entries) { $target[$copiedKey] = $copiedValue }; $target as $valueType }" + } + else -> "copyContext.copyObject($expression) as $valueType" } - else -> "copyContext.copyObject($expression) as $valueType" - } + return applyCollectionFactory(type, copied) } private fun copyElementExpression( @@ -1125,9 +1221,11 @@ internal class KotlinSerializerSourceWriter(private val struct: KotlinSourceStru val value = "copyValue$depth" return "$expression?.let { $value -> ${copyElementExpression(type.copy(nullable = false), value, depth + 1)} }" } - if ( - type.primitive || type.unsigned || type.typeId == "Types.STRING" || type.componentType != null - ) { + val arrayCopy = copyArrayExpression(type, expression) + if (arrayCopy != null) { + return arrayCopy + } + if (type.primitive || type.unsigned || type.typeId == "Types.STRING") { return expression } if (type.isCollectionOrMap()) { @@ -1136,6 +1234,30 @@ internal class KotlinSerializerSourceWriter(private val struct: KotlinSourceStru return "(copyContext.copyObject($expression) as ${type.valueTypeName.removeSuffix("?")})" } + private fun copyArrayExpression(type: KotlinSourceTypeNode, expression: String): String? { + val valueType = type.valueTypeName.removeSuffix("?") + if (type.typeId == "Types.BINARY" && valueType == "ByteArray") { + return "$expression.copyOf()" + } + if (type.componentType == null) { + return null + } + return when (valueType) { + "BooleanArray", + "ByteArray", + "ShortArray", + "IntArray", + "LongArray", + "FloatArray", + "DoubleArray", + "UByteArray", + "UShortArray", + "UIntArray", + "ULongArray" -> "$expression.copyOf()" + else -> "(copyContext.copyObject($expression) as $valueType)" + } + } + private fun fromJavaCompatExpr( type: KotlinSourceTypeNode, expression: String, diff --git a/kotlin/fory-kotlin-ksp/src/test/kotlin/org/apache/fory/kotlin/ksp/ProcessorValidationTest.kt b/kotlin/fory-kotlin-ksp/src/test/kotlin/org/apache/fory/kotlin/ksp/ProcessorValidationTest.kt index ab2e56df1d..6c62579090 100644 --- a/kotlin/fory-kotlin-ksp/src/test/kotlin/org/apache/fory/kotlin/ksp/ProcessorValidationTest.kt +++ b/kotlin/fory-kotlin-ksp/src/test/kotlin/org/apache/fory/kotlin/ksp/ProcessorValidationTest.kt @@ -174,6 +174,59 @@ class ProcessorValidationTest { primitive = true, unsigned = false, ) + val intArrayType = + KotlinSourceTypeNode( + rawClassExpression = "IntArray::class.java", + kotlinTypeName = "kotlin.IntArray", + valueTypeName = "IntArray", + typeName = "int[]", + typeId = "Types.INT32_ARRAY", + nullable = false, + trackingRef = false, + primitive = false, + unsigned = false, + componentType = intType, + ) + val setType = + KotlinSourceTypeNode( + rawClassExpression = "java.util.Set::class.java", + kotlinTypeName = "java.util.TreeSet", + valueTypeName = "kotlin.collections.Set", + typeName = "java.util.Set", + typeId = "Types.SET", + nullable = false, + trackingRef = false, + primitive = false, + unsigned = false, + collectionFactory = CollectionFactory.TREE_SET, + typeArguments = listOf(stringType), + ) + val listType = + KotlinSourceTypeNode( + rawClassExpression = "java.util.List::class.java", + kotlinTypeName = "java.util.List>", + valueTypeName = "kotlin.collections.List>", + typeName = "java.util.List", + typeId = "Types.LIST", + nullable = false, + trackingRef = false, + primitive = false, + unsigned = false, + typeArguments = listOf(setType), + ) + val arrayListType = + KotlinSourceTypeNode( + rawClassExpression = "java.util.List::class.java", + kotlinTypeName = "java.util.List", + valueTypeName = "kotlin.collections.List", + typeName = "java.util.List", + typeId = "Types.LIST", + nullable = false, + trackingRef = false, + primitive = false, + unsigned = false, + typeArguments = listOf(intArrayType), + ) val mapType = KotlinSourceTypeNode( rawClassExpression = "java.util.Map::class.java", @@ -211,6 +264,34 @@ class ProcessorValidationTest { nullable = false, propertyTypeName = "java.util.TreeMap", constructorParameterName = "counts", + ), + KotlinSourceField( + id = 1, + name = "names", + type = listType, + hasForyField = true, + foryFieldId = 2, + trackingRef = false, + dynamic = "AUTO", + arrayType = false, + hasDefault = false, + nullable = false, + propertyTypeName = "java.util.List>", + constructorParameterName = "names", + ), + KotlinSourceField( + id = 2, + name = "arrays", + type = arrayListType, + hasForyField = true, + foryFieldId = 3, + trackingRef = false, + dynamic = "AUTO", + arrayType = false, + hasDefault = false, + nullable = false, + propertyTypeName = "java.util.List", + constructorParameterName = "arrays", ) ), originatingFiles = emptyList(), @@ -219,23 +300,45 @@ class ProcessorValidationTest { .write() assertTrue( - source.contains("return User(counts = KotlinCollectionAdapters.toTreeMap(field0!!))") + source.contains( + "return User(counts = KotlinCollectionAdapters.toTreeMap(field0!!), names = run { val readSource0 = (field1!! as Collection<*>);" + ) + ) + assertTrue( + source.contains("KotlinCollectionAdapters.toTreeSet((readElement0 as Collection<*>))") ) assertTrue( source.contains( "0 -> KotlinCollectionAdapters.toTreeMap((readFieldValue(readContext, fieldInfo) as kotlin.collections.Map))" ) ) + assertTrue( + source.contains( + "1 -> run { val readSource0 = ((readFieldValue(readContext, fieldInfo) as kotlin.collections.List>) as Collection<*>);" + ) + ) assertTrue( source.contains( "0 -> KotlinCollectionAdapters.toTreeMap((readCompatibleFieldValue(readContext, remoteField, localField) as kotlin.collections.Map))" ) ) + assertTrue( + source.contains( + "1 -> run { val readSource0 = ((readCompatibleFieldValue(readContext, remoteField, localField) as kotlin.collections.List>) as Collection<*>);" + ) + ) assertTrue( source.contains( "fieldValues[0] = KotlinCollectionAdapters.toTreeMap(run { val copySource0 = value.counts; val copyTarget0 = java.util.LinkedHashMap(copySource0.size);" ) ) + assertTrue( + source.contains( + "fieldValues[1] = run { val copySource0 = value.names; val copyTarget0 = java.util.ArrayList(copySource0.size); for (copyElement0 in copySource0) { copyTarget0.add(KotlinCollectionAdapters.toTreeSet(run { val copySource2 = copyElement0;" + ) + ) + assertTrue(source.contains("fieldValues[2] = run { val copySource0 = value.arrays;")) + assertTrue(source.contains("copyTarget0.add(copyElement0.copyOf())")) assertFalse(source.contains("fieldValues[0] = copyConstructorFieldValue")) } diff --git a/kotlin/fory-kotlin-tests/src/main/kotlin/org/apache/fory/kotlin/xlang/KotlinXlangPeer.kt b/kotlin/fory-kotlin-tests/src/main/kotlin/org/apache/fory/kotlin/xlang/KotlinXlangPeer.kt index caaf24756a..a9a62767af 100644 --- a/kotlin/fory-kotlin-tests/src/main/kotlin/org/apache/fory/kotlin/xlang/KotlinXlangPeer.kt +++ b/kotlin/fory-kotlin-tests/src/main/kotlin/org/apache/fory/kotlin/xlang/KotlinXlangPeer.kt @@ -126,6 +126,7 @@ public data class KotlinSchemaSurface "noRefUsers", "chunks", "chunksByName", + "nestedSortedNames", ) constructor( @ForyField(id = 1) val nullableNames: List?, @@ -140,6 +141,7 @@ constructor( @ForyField(id = 10) val noRefUsers: List, @ForyField(id = 11) val chunks: List<@ArrayType ByteArray>, @ForyField(id = 12) val chunksByName: Map, + @ForyField(id = 13) val nestedSortedNames: List>, ) @ForyStruct @@ -341,6 +343,7 @@ private fun staticSerializerRoundTrip(dataFile: String) { noRefUsers = emptyList(), chunks = listOf(byteArrayOf(1, 2), byteArrayOf(3, 4)), chunksByName = mapOf("left" to byteArrayOf(5, 6)), + nestedSortedNames = listOf(TreeSet(listOf("blue", "green"))), ) val decodedSchemaSurface = fory.deserialize(fory.serialize(schemaSurface), KotlinSchemaSurface::class.java) @@ -362,6 +365,25 @@ private fun staticSerializerRoundTrip(dataFile: String) { checkNotNull(decodedSchemaSurface.chunksByName["left"]) contentEquals schemaSurface.chunksByName["left"]!! ) + check(decodedSchemaSurface.nestedSortedNames == schemaSurface.nestedSortedNames) + check(decodedSchemaSurface.nestedSortedNames[0].javaClass == TreeSet::class.java) + val copiedSchemaSurface = fory.copy(schemaSurface) + check(copiedSchemaSurface.bytesAsArray !== schemaSurface.bytesAsArray) + check(copiedSchemaSurface.chunks !== schemaSurface.chunks) + check(copiedSchemaSurface.chunks[0] !== schemaSurface.chunks[0]) + check(copiedSchemaSurface.chunksByName !== schemaSurface.chunksByName) + check( + checkNotNull(copiedSchemaSurface.chunksByName["left"]) !== schemaSurface.chunksByName["left"] + ) + check(copiedSchemaSurface.nestedSortedNames !== schemaSurface.nestedSortedNames) + check(copiedSchemaSurface.nestedSortedNames[0] !== schemaSurface.nestedSortedNames[0]) + check(copiedSchemaSurface.nestedSortedNames[0].javaClass == TreeSet::class.java) + schemaSurface.bytesAsArray[0] = 99.toByte() + schemaSurface.chunks[0][0] = 98.toByte() + checkNotNull(schemaSurface.chunksByName["left"])[0] = 97.toByte() + check(copiedSchemaSurface.bytesAsArray[0] == 1.toByte()) + check(copiedSchemaSurface.chunks[0][0] == 1.toByte()) + check(checkNotNull(copiedSchemaSurface.chunksByName["left"])[0] == 5.toByte()) val schemaDescriptors = checkNotNull( fory.getSerializer(KotlinSchemaSurface::class.java) as? StaticGeneratedStructSerializer<*> From 37dee621b1b9449e548539fae4c285ba897385d6 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Tue, 2 Jun 2026 14:32:20 +0800 Subject: [PATCH 41/69] refactor(java): isolate unsafe access for jdk25 --- .agents/languages/java.md | 10 + .../org/apache/fory/memory/LittleEndian.java | 4 +- .../org/apache/fory/memory/MemoryBuffer.java | 4 +- .../fory/platform/internal/_JDKAccess.java | 370 +------- .../fory/platform/internal/_Lookup.java | 6 +- .../fory/platform/internal/_UnsafeUtils.java | 42 + .../fory/reflect/FieldAccessorStrategy.java | 3 +- .../fory/reflect/UnsafeObjectAllocator.java | 4 +- .../apache/fory/resolver/XtypeResolver.java | 44 +- .../serializer/ObjectStreamSerializer.java | 10 +- .../fory/serializer/PlatformStringUtils.java | 160 ++-- .../serializer/ReplaceResolveSerializer.java | 6 +- .../serializer/SerializationHookLookup.java | 131 +++ .../SerializedLambdaSerializer.java | 4 +- .../fory/serializer/StringSerializer.java | 135 ++- .../collection/CollectionSerializers.java | 27 +- .../serializer/collection/MapSerializers.java | 22 +- .../org/apache/fory/memory/MemoryBuffer.java | 723 +++++----------- .../fory/platform/internal/DefineClass.java | 9 - .../fory/platform/internal/_JDKAccess.java | 809 ------------------ .../fory/serializer/PlatformStringUtils.java | 160 +++- .../serializer/SerializationHookLookup.java | 143 ++++ .../fory-core/native-image.properties | 2 +- .../fory/serializer/StringSerializerTest.java | 3 +- .../collection/CollectionSerializersTest.java | 105 ++- .../collection/MapSerializersTest.java | 93 +- .../ksp/KotlinSerializerSourceWriter.kt | 367 +++++++- .../kotlin/ksp/ProcessorValidationTest.kt | 210 ++++- .../fory/kotlin/xlang/KotlinXlangPeer.kt | 59 +- 29 files changed, 1808 insertions(+), 1857 deletions(-) create mode 100644 java/fory-core/src/main/java/org/apache/fory/platform/internal/_UnsafeUtils.java create mode 100644 java/fory-core/src/main/java/org/apache/fory/serializer/SerializationHookLookup.java delete mode 100644 java/fory-core/src/main/java25/org/apache/fory/platform/internal/_JDKAccess.java create mode 100644 java/fory-core/src/main/java25/org/apache/fory/serializer/SerializationHookLookup.java diff --git a/.agents/languages/java.md b/.agents/languages/java.md index 310c78784f..1d659fa984 100644 --- a/.agents/languages/java.md +++ b/.agents/languages/java.md @@ -76,6 +76,13 @@ Load this file when changing anything under `java/` or when Java drives a cross- - Root codegen and builder classes that still need Unsafe on JDK8-24 must route symbolic Unsafe access through a helper with a Java 25 replacement. Do not leave `_JDKAccess.unsafe()` or `sun.misc.Unsafe` references in JDK25-visible classes outside matching `java25` replacements. +- Java 25 multi-release classes never run on Android. Do not keep `AndroidSupport.IS_ANDROID` + branches or imports under `src/main/java25`; Android compatibility belongs to the root sources. +- String zero-copy construction is serializer-owned behavior. Keep private `String` constructor + lambda factories in `StringSerializer`, keep `PlatformStringUtils` + focused on field and array access, keep serialization hook discovery in serializer-owned code, + and keep `_JDKAccess` limited to JDK lookup, module, function factory, and access-flag + primitives. - JDK25+ collection serializers must fail unsupported `Collections.newSetFromMap` backing maps before writing or copying. Do not rewrite them to `HashMap`, because that changes equality semantics and can drop entries. @@ -89,6 +96,9 @@ Load this file when changing anything under `java/` or when Java drives a cross- reject copied constructor arguments that still retain the marker, then construct and reference the real copy. Do not implement a separate raw-field pre-scan or leave a generated path without the marker guard. +- Generated JVM copy code may direct-copy immutable scalar values, but Java `Collection`/`Map` + subclasses must be copied through `CopyContext.copyObject(...)` so collection/map serializers own + concrete type, comparator, wrapper, and reference behavior. - When a `Throwable` is created without running `Throwable` constructors, restore the private `cause` and `suppressedExceptions` sentinels directly before exposing the object. Do not call `initCause` or `addSuppressed` on a constructor-bypassed `Throwable` whose sentinels are still diff --git a/java/fory-core/src/main/java/org/apache/fory/memory/LittleEndian.java b/java/fory-core/src/main/java/org/apache/fory/memory/LittleEndian.java index a2105ef17e..4cb7b5ef6a 100644 --- a/java/fory-core/src/main/java/org/apache/fory/memory/LittleEndian.java +++ b/java/fory-core/src/main/java/org/apache/fory/memory/LittleEndian.java @@ -1,7 +1,7 @@ package org.apache.fory.memory; import org.apache.fory.platform.AndroidSupport; -import org.apache.fory.platform.internal._JDKAccess; +import org.apache.fory.platform.internal._UnsafeUtils; import sun.misc.Unsafe; /* @@ -24,7 +24,7 @@ */ public class LittleEndian { - private static final Unsafe UNSAFE = AndroidSupport.IS_ANDROID ? null : _JDKAccess.UNSAFE; + private static final Unsafe UNSAFE = AndroidSupport.IS_ANDROID ? null : _UnsafeUtils.UNSAFE; private static final int BYTE_ARRAY_OFFSET; // Keep arrayBaseOffset as a direct static-field store for GraalVM native-image recomputation. diff --git a/java/fory-core/src/main/java/org/apache/fory/memory/MemoryBuffer.java b/java/fory-core/src/main/java/org/apache/fory/memory/MemoryBuffer.java index 5b0e73617d..04733e0453 100644 --- a/java/fory-core/src/main/java/org/apache/fory/memory/MemoryBuffer.java +++ b/java/fory-core/src/main/java/org/apache/fory/memory/MemoryBuffer.java @@ -30,7 +30,7 @@ import org.apache.fory.io.ForyStreamReader; import org.apache.fory.platform.AndroidSupport; import org.apache.fory.platform.JdkVersion; -import org.apache.fory.platform.internal._JDKAccess; +import org.apache.fory.platform.internal._UnsafeUtils; import sun.misc.Unsafe; /** @@ -66,7 +66,7 @@ */ public final class MemoryBuffer { public static final int BUFFER_GROW_STEP_THRESHOLD = 100 * 1024 * 1024; - private static final Unsafe UNSAFE = AndroidSupport.IS_ANDROID ? null : _JDKAccess.UNSAFE; + private static final Unsafe UNSAFE = AndroidSupport.IS_ANDROID ? null : _UnsafeUtils.UNSAFE; private static final boolean LITTLE_ENDIAN = NativeByteOrder.IS_LITTLE_ENDIAN; private static final boolean UNALIGNED = !AndroidSupport.IS_ANDROID && unaligned(); private static final int BOOLEAN_ARRAY_OFFSET; diff --git a/java/fory-core/src/main/java/org/apache/fory/platform/internal/_JDKAccess.java b/java/fory-core/src/main/java/org/apache/fory/platform/internal/_JDKAccess.java index 54a7259aa5..635b61f1d5 100644 --- a/java/fory-core/src/main/java/org/apache/fory/platform/internal/_JDKAccess.java +++ b/java/fory-core/src/main/java/org/apache/fory/platform/internal/_JDKAccess.java @@ -25,14 +25,11 @@ import java.lang.invoke.MethodHandles; import java.lang.invoke.MethodHandles.Lookup; import java.lang.invoke.MethodType; -import java.lang.reflect.Field; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; -import java.nio.charset.StandardCharsets; import java.util.HashMap; import java.util.Map; import java.util.function.BiConsumer; -import java.util.function.BiFunction; import java.util.function.Consumer; import java.util.function.Function; import java.util.function.Predicate; @@ -51,156 +48,53 @@ import org.apache.fory.util.function.ToCharFunction; import org.apache.fory.util.function.ToFloatFunction; import org.apache.fory.util.function.ToShortFunction; -import sun.misc.Unsafe; -/** Unsafe JDK utils. */ +/** JDK lookup, module, and lambda factory utils. */ // CHECKSTYLE.OFF:TypeName public class _JDKAccess { // CHECKSTYLE.ON:TypeName public static final boolean IS_OPEN_J9; - public static final Unsafe UNSAFE; - // Root classes use Unsafe for JDK internal fields. A JDK25 multi-release _JDKAccess must keep - // this API surface and implement supported cases with VarHandle, or set this false so callers - // choose public fallbacks. public static final boolean JDK_INTERNAL_FIELD_ACCESS; public static final boolean JDK_LANG_FIELD_ACCESS; - public static final boolean JDK_STRING_FIELD_ACCESS; public static final boolean JDK_COLLECTION_FIELD_ACCESS; public static final boolean JDK_CONCURRENT_FIELD_ACCESS; public static final boolean JDK_PROXY_FIELD_ACCESS; - public static final Class _INNER_UNSAFE_CLASS; - public static final Object _INNER_UNSAFE; static { String jmvName = System.getProperty("java.vm.name", ""); IS_OPEN_J9 = jmvName.contains("OpenJ9"); - Unsafe unsafe; - try { - Field unsafeField = Unsafe.class.getDeclaredField("theUnsafe"); - unsafeField.setAccessible(true); - unsafe = (Unsafe) unsafeField.get(null); - } catch (Throwable cause) { - throw new UnsupportedOperationException("Unsafe is not supported in this platform."); - } - UNSAFE = unsafe; if (AndroidSupport.IS_ANDROID) { JDK_INTERNAL_FIELD_ACCESS = false; JDK_LANG_FIELD_ACCESS = false; - JDK_STRING_FIELD_ACCESS = false; JDK_COLLECTION_FIELD_ACCESS = false; JDK_CONCURRENT_FIELD_ACCESS = false; JDK_PROXY_FIELD_ACCESS = false; + } else if (JdkVersion.MAJOR_VERSION >= 25) { + boolean trustedLookupAvailable = trustedLookupAvailable(); + JDK_INTERNAL_FIELD_ACCESS = trustedLookupAvailable; + JDK_LANG_FIELD_ACCESS = trustedLookupAvailable; + JDK_COLLECTION_FIELD_ACCESS = trustedLookupAvailable; + JDK_CONCURRENT_FIELD_ACCESS = trustedLookupAvailable; + JDK_PROXY_FIELD_ACCESS = trustedLookupAvailable; } else { JDK_INTERNAL_FIELD_ACCESS = true; JDK_LANG_FIELD_ACCESS = true; - JDK_STRING_FIELD_ACCESS = true; JDK_COLLECTION_FIELD_ACCESS = true; JDK_CONCURRENT_FIELD_ACCESS = true; JDK_PROXY_FIELD_ACCESS = true; } - Object innerUnsafe = null; - Class innerUnsafeClass = null; - if (!AndroidSupport.IS_ANDROID && JdkVersion.MAJOR_VERSION >= 11) { - try { - Field theInternalUnsafeField = Unsafe.class.getDeclaredField("theInternalUnsafe"); - theInternalUnsafeField.setAccessible(true); - innerUnsafe = theInternalUnsafeField.get(null); - innerUnsafeClass = innerUnsafe.getClass(); - } catch (Exception e) { - throw new RuntimeException(e); - } - } - _INNER_UNSAFE = innerUnsafe; - _INNER_UNSAFE_CLASS = innerUnsafeClass; - } - - public static Unsafe unsafe() { - return UNSAFE; } - private static final ClassValueCache lookupCache = ClassValueCache.newClassKeyCache(32); - - public static final boolean STRING_VALUE_FIELD_IS_CHARS; - public static final boolean STRING_VALUE_FIELD_IS_BYTES; - public static final boolean STRING_HAS_COUNT_OFFSET; - public static final long STRING_VALUE_FIELD_OFFSET; - public static final long STRING_COUNT_FIELD_OFFSET; - public static final long STRING_OFFSET_FIELD_OFFSET; - - static { - if (AndroidSupport.IS_ANDROID) { - STRING_VALUE_FIELD_IS_CHARS = false; - STRING_VALUE_FIELD_IS_BYTES = false; - STRING_VALUE_FIELD_OFFSET = -1; - STRING_HAS_COUNT_OFFSET = false; - STRING_COUNT_FIELD_OFFSET = -1; - STRING_OFFSET_FIELD_OFFSET = -1; - } else { - try { - Field valueField = String.class.getDeclaredField("value"); - STRING_VALUE_FIELD_IS_CHARS = valueField.getType() == char[].class; - STRING_VALUE_FIELD_IS_BYTES = valueField.getType() == byte[].class; - STRING_VALUE_FIELD_OFFSET = UNSAFE.objectFieldOffset(valueField); - Field countField = getStringFieldNullable("count"); - Field offsetField = getStringFieldNullable("offset"); - if (countField != null || offsetField != null) { - Preconditions.checkArgument( - countField != null && offsetField != null, "Current jdk not supported"); - Preconditions.checkArgument( - countField.getType() == int.class && offsetField.getType() == int.class, - "Current jdk not supported"); - STRING_HAS_COUNT_OFFSET = true; - STRING_COUNT_FIELD_OFFSET = UNSAFE.objectFieldOffset(countField); - STRING_OFFSET_FIELD_OFFSET = UNSAFE.objectFieldOffset(offsetField); - } else { - STRING_HAS_COUNT_OFFSET = false; - STRING_COUNT_FIELD_OFFSET = -1; - STRING_OFFSET_FIELD_OFFSET = -1; - } - } catch (NoSuchFieldException e) { - throw new RuntimeException(e); - } - } - } - - private static Field getStringFieldNullable(String fieldName) { + private static boolean trustedLookupAvailable() { try { - return String.class.getDeclaredField(fieldName); - } catch (NoSuchFieldException e) { - return null; + _Lookup._trustedLookup(Object.class); + return true; + } catch (Throwable ignored) { + return false; } } - private static class StringCoderField { - private static final long OFFSET; - - static { - try { - OFFSET = UNSAFE.objectFieldOffset(String.class.getDeclaredField("coder")); - } catch (NoSuchFieldException e) { - throw new RuntimeException(e); - } - } - } - - public static final long STRING_CODER_FIELD_OFFSET = - STRING_VALUE_FIELD_IS_BYTES ? StringCoderField.OFFSET : -1; - - public static Object getStringValue(String value) { - return UNSAFE.getObject(value, STRING_VALUE_FIELD_OFFSET); - } - - public static byte getStringCoder(String value) { - return UNSAFE.getByte(value, STRING_CODER_FIELD_OFFSET); - } - - public static int getStringOffset(String value) { - return UNSAFE.getInt(value, STRING_OFFSET_FIELD_OFFSET); - } - - public static int getStringCount(String value) { - return UNSAFE.getInt(value, STRING_COUNT_FIELD_OFFSET); - } + private static final ClassValueCache lookupCache = ClassValueCache.newClassKeyCache(32); // CHECKSTYLE.OFF:MethodName @@ -214,242 +108,6 @@ public static Lookup _trustedLookup(Class objectClass) { return lookupCache.get(objectClass, () -> _Lookup._trustedLookup(objectClass)); } - public static MethodHandle readResolveHandle(Class cls, Method method) - throws IllegalAccessException { - return _trustedLookup(cls).unreflect(method); - } - - private static final byte LATIN1 = 0; - private static final Byte LATIN1_BOXED = LATIN1; - private static final byte UTF16 = 1; - private static final Byte UTF16_BOXED = UTF16; - private static final MethodHandles.Lookup STRING_LOOK_UP = - JDK_INTERNAL_FIELD_ACCESS ? _trustedLookup(String.class) : null; - private static final BiFunction CHARS_STRING_ZERO_COPY_CTR = - JDK_INTERNAL_FIELD_ACCESS ? getCharsStringZeroCopyCtr() : null; - private static final BiFunction BYTES_STRING_ZERO_COPY_CTR = - JDK_INTERNAL_FIELD_ACCESS ? getBytesStringZeroCopyCtr() : null; - private static final Function LATIN_BYTES_STRING_ZERO_COPY_CTR = - JDK_INTERNAL_FIELD_ACCESS ? getLatinBytesStringZeroCopyCtr() : null; - - public static String newCharsStringZeroCopy(char[] data) { - if (!JDK_INTERNAL_FIELD_ACCESS) { - return new String(data); - } - if (!STRING_VALUE_FIELD_IS_CHARS) { - throw new IllegalStateException("String value isn't char[], current java isn't supported"); - } - return CHARS_STRING_ZERO_COPY_CTR.apply(data, Boolean.TRUE); - } - - public static String newBytesStringZeroCopy(byte coder, byte[] data) { - if (!JDK_INTERNAL_FIELD_ACCESS) { - return newBytesStringSlow(coder, data); - } - if (coder == LATIN1) { - if (LATIN_BYTES_STRING_ZERO_COPY_CTR != null) { - return LATIN_BYTES_STRING_ZERO_COPY_CTR.apply(data); - } - return BYTES_STRING_ZERO_COPY_CTR.apply(data, LATIN1_BOXED); - } else if (coder == UTF16) { - return BYTES_STRING_ZERO_COPY_CTR.apply(data, UTF16_BOXED); - } else { - return BYTES_STRING_ZERO_COPY_CTR.apply(data, coder); - } - } - - private static String newBytesStringSlow(byte coder, byte[] data) { - if (coder == LATIN1) { - return new String(data, StandardCharsets.ISO_8859_1); - } else if (coder == UTF16) { - char[] chars = new char[data.length >> 1]; - for (int i = 0, j = 0; i < data.length; i += 2) { - chars[j++] = (char) ((data[i] & 0xff) | ((data[i + 1] & 0xff) << 8)); - } - return new String(chars); - } else { - return new String(data, StandardCharsets.UTF_8); - } - } - - private static BiFunction getCharsStringZeroCopyCtr() { - if (!STRING_VALUE_FIELD_IS_CHARS) { - return null; - } - MethodHandle handle = getJavaStringZeroCopyCtrHandle(); - if (handle == null) { - return null; - } - try { - CallSite callSite = - LambdaMetafactory.metafactory( - STRING_LOOK_UP, - "apply", - MethodType.methodType(BiFunction.class), - handle.type().generic(), - handle, - handle.type()); - return (BiFunction) callSite.getTarget().invokeExact(); - } catch (Throwable e) { - return null; - } - } - - private static BiFunction getBytesStringZeroCopyCtr() { - if (!STRING_VALUE_FIELD_IS_BYTES) { - return null; - } - MethodHandle handle = getJavaStringZeroCopyCtrHandle(); - if (handle == null) { - return null; - } - try { - MethodType instantiatedMethodType = - MethodType.methodType(handle.type().returnType(), new Class[] {byte[].class, Byte.class}); - CallSite callSite = - LambdaMetafactory.metafactory( - STRING_LOOK_UP, - "apply", - MethodType.methodType(BiFunction.class), - handle.type().generic(), - handle, - instantiatedMethodType); - return (BiFunction) callSite.getTarget().invokeExact(); - } catch (Throwable e) { - return null; - } - } - - private static Function getLatinBytesStringZeroCopyCtr() { - if (!STRING_VALUE_FIELD_IS_BYTES || STRING_LOOK_UP == null) { - return null; - } - try { - Class clazz = Class.forName("java.lang.StringCoding"); - Lookup caller = STRING_LOOK_UP.in(clazz); - MethodHandle handle = - caller.findStatic( - clazz, "newStringLatin1", MethodType.methodType(String.class, byte[].class)); - return makeFunction(caller, handle, Function.class); - } catch (Throwable e) { - return null; - } - } - - private static MethodHandle getJavaStringZeroCopyCtrHandle() { - Preconditions.checkArgument(JdkVersion.MAJOR_VERSION >= 8); - if (STRING_LOOK_UP == null) { - return null; - } - try { - if (STRING_VALUE_FIELD_IS_CHARS) { - return STRING_LOOK_UP.findConstructor( - String.class, MethodType.methodType(void.class, char[].class, boolean.class)); - } else { - return STRING_LOOK_UP.findConstructor( - String.class, MethodType.methodType(void.class, byte[].class, byte.class)); - } - } catch (Exception e) { - return null; - } - } - - private static class SerializationMethods { - private static final Object REFLECTION_FACTORY; - private static final Method WRITE_OBJECT; - private static final Method READ_OBJECT; - private static final Method READ_OBJECT_NO_DATA; - private static final Method DEFAULT_READ_OBJECT; - private static final Method WRITE_REPLACE; - private static final Method READ_RESOLVE; - - static { - Object reflectionFactory = null; - Method writeObject = null; - Method readObject = null; - Method readObjectNoData = null; - Method defaultReadObject = null; - Method writeReplace = null; - Method readResolve = null; - try { - Class factoryClass = Class.forName("sun.reflect.ReflectionFactory"); - Method getReflectionFactory = factoryClass.getDeclaredMethod("getReflectionFactory"); - reflectionFactory = getReflectionFactory.invoke(null); - writeObject = factoryClass.getDeclaredMethod("writeObjectForSerialization", Class.class); - readObject = factoryClass.getDeclaredMethod("readObjectForSerialization", Class.class); - readObjectNoData = - factoryClass.getDeclaredMethod("readObjectNoDataForSerialization", Class.class); - try { - defaultReadObject = - factoryClass.getDeclaredMethod("defaultReadObjectForSerialization", Class.class); - } catch (NoSuchMethodException e) { - ExceptionUtils.ignore(e); - } - writeReplace = factoryClass.getDeclaredMethod("writeReplaceForSerialization", Class.class); - readResolve = factoryClass.getDeclaredMethod("readResolveForSerialization", Class.class); - } catch (Throwable e) { - ExceptionUtils.ignore(e); - } - REFLECTION_FACTORY = reflectionFactory; - WRITE_OBJECT = writeObject; - READ_OBJECT = readObject; - READ_OBJECT_NO_DATA = readObjectNoData; - DEFAULT_READ_OBJECT = defaultReadObject; - WRITE_REPLACE = writeReplace; - READ_RESOLVE = readResolve; - } - } - - private static MethodHandle getSerializationHandle(Class type, Method factoryMethod) { - if (SerializationMethods.REFLECTION_FACTORY == null || factoryMethod == null) { - return null; - } - try { - return (MethodHandle) factoryMethod.invoke(SerializationMethods.REFLECTION_FACTORY, type); - } catch (Throwable e) { - ExceptionUtils.ignore(e); - return null; - } - } - - private static Method getSerializationMethod(Class type, Method factoryMethod) { - MethodHandle handle = getSerializationHandle(type, factoryMethod); - return handle == null ? null : MethodHandles.reflectAs(Method.class, handle); - } - - public static Method getSerializationWriteObjectMethod(Class type) { - return getSerializationMethod(type, SerializationMethods.WRITE_OBJECT); - } - - public static Method getSerializationReadObjectMethod(Class type) { - return getSerializationMethod(type, SerializationMethods.READ_OBJECT); - } - - public static Method getSerializationReadObjectNoDataMethod(Class type) { - return getSerializationMethod(type, SerializationMethods.READ_OBJECT_NO_DATA); - } - - public static MethodHandle getSerializationDefaultReadObjectHandle(Class type) { - return getSerializationHandle(type, SerializationMethods.DEFAULT_READ_OBJECT); - } - - public static Method getSerializationWriteReplaceMethod(Class type) { - return getSerializationMethod(type, SerializationMethods.WRITE_REPLACE); - } - - public static Method getSerializationReadResolveMethod(Class type) { - return getSerializationMethod(type, SerializationMethods.READ_RESOLVE); - } - - public static boolean isSerializationHookLookupAvailable() { - return SerializationMethods.REFLECTION_FACTORY != null - && SerializationMethods.WRITE_OBJECT != null - && SerializationMethods.READ_OBJECT != null - && SerializationMethods.READ_OBJECT_NO_DATA != null - && SerializationMethods.WRITE_REPLACE != null - && SerializationMethods.READ_RESOLVE != null; - } - public static T tryMakeFunction( Lookup lookup, MethodHandle handle, Class functionInterface) { try { diff --git a/java/fory-core/src/main/java/org/apache/fory/platform/internal/_Lookup.java b/java/fory-core/src/main/java/org/apache/fory/platform/internal/_Lookup.java index 2ece8bb9ce..505e74549c 100644 --- a/java/fory-core/src/main/java/org/apache/fory/platform/internal/_Lookup.java +++ b/java/fory-core/src/main/java/org/apache/fory/platform/internal/_Lookup.java @@ -44,9 +44,9 @@ class _Lookup { { try { Field implLookup = Lookup.class.getDeclaredField("IMPL_LOOKUP"); - long fieldOffset = _JDKAccess.UNSAFE.staticFieldOffset(implLookup); - Object fieldBase = _JDKAccess.UNSAFE.staticFieldBase(implLookup); - trustedLookup = (Lookup) _JDKAccess.UNSAFE.getObject(fieldBase, fieldOffset); + long fieldOffset = _UnsafeUtils.UNSAFE.staticFieldOffset(implLookup); + Object fieldBase = _UnsafeUtils.UNSAFE.staticFieldBase(implLookup); + trustedLookup = (Lookup) _UnsafeUtils.UNSAFE.getObject(fieldBase, fieldOffset); } catch (Throwable ignored) { // ignored } diff --git a/java/fory-core/src/main/java/org/apache/fory/platform/internal/_UnsafeUtils.java b/java/fory-core/src/main/java/org/apache/fory/platform/internal/_UnsafeUtils.java new file mode 100644 index 0000000000..32555f842e --- /dev/null +++ b/java/fory-core/src/main/java/org/apache/fory/platform/internal/_UnsafeUtils.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.platform.internal; + +import java.lang.reflect.Field; +import sun.misc.Unsafe; + +/** Root-runtime owner for {@link Unsafe}. Java25+ code must use overlay classes instead. */ +// CHECKSTYLE.OFF:TypeName +public final class _UnsafeUtils { + // CHECKSTYLE.ON:TypeName + public static final Unsafe UNSAFE; + + static { + try { + Field unsafeField = Unsafe.class.getDeclaredField("theUnsafe"); + unsafeField.setAccessible(true); + UNSAFE = (Unsafe) unsafeField.get(null); + } catch (Throwable cause) { + throw new UnsupportedOperationException("Unsafe is not supported in this platform."); + } + } + + private _UnsafeUtils() {} +} diff --git a/java/fory-core/src/main/java/org/apache/fory/reflect/FieldAccessorStrategy.java b/java/fory-core/src/main/java/org/apache/fory/reflect/FieldAccessorStrategy.java index 22a503cf90..23d4ebb535 100644 --- a/java/fory-core/src/main/java/org/apache/fory/reflect/FieldAccessorStrategy.java +++ b/java/fory-core/src/main/java/org/apache/fory/reflect/FieldAccessorStrategy.java @@ -30,12 +30,13 @@ import org.apache.fory.platform.AndroidSupport; import org.apache.fory.platform.GraalvmSupport; import org.apache.fory.platform.internal._JDKAccess; +import org.apache.fory.platform.internal._UnsafeUtils; import org.apache.fory.type.TypeUtils; import org.apache.fory.util.Preconditions; import sun.misc.Unsafe; final class FieldAccessorStrategy { - private static final Unsafe UNSAFE = AndroidSupport.IS_ANDROID ? null : _JDKAccess.UNSAFE; + private static final Unsafe UNSAFE = AndroidSupport.IS_ANDROID ? null : _UnsafeUtils.UNSAFE; private static final int BOOLEAN_ACCESS = 1; private static final int BYTE_ACCESS = 2; diff --git a/java/fory-core/src/main/java/org/apache/fory/reflect/UnsafeObjectAllocator.java b/java/fory-core/src/main/java/org/apache/fory/reflect/UnsafeObjectAllocator.java index 0e9d1b0dd3..f1e14b79b0 100644 --- a/java/fory-core/src/main/java/org/apache/fory/reflect/UnsafeObjectAllocator.java +++ b/java/fory-core/src/main/java/org/apache/fory/reflect/UnsafeObjectAllocator.java @@ -23,13 +23,13 @@ import org.apache.fory.exception.ForyException; import org.apache.fory.platform.AndroidSupport; import org.apache.fory.platform.JdkVersion; -import org.apache.fory.platform.internal._JDKAccess; +import org.apache.fory.platform.internal._UnsafeUtils; import sun.misc.Unsafe; /** Internal JDK8-24 allocator used by object creators. */ @Internal final class UnsafeObjectAllocator { - private static final Unsafe UNSAFE = AndroidSupport.IS_ANDROID ? null : _JDKAccess.UNSAFE; + private static final Unsafe UNSAFE = AndroidSupport.IS_ANDROID ? null : _UnsafeUtils.UNSAFE; private UnsafeObjectAllocator() {} diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java index 9972145d3c..d17bbc1078 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java @@ -41,9 +41,17 @@ import java.util.HashSet; import java.util.LinkedHashMap; import java.util.LinkedHashSet; +import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.TreeMap; +import java.util.TreeSet; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentSkipListMap; +import java.util.concurrent.ConcurrentSkipListSet; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.CopyOnWriteArraySet; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; @@ -103,11 +111,20 @@ import org.apache.fory.serializer.collection.CollectionLikeSerializer; import org.apache.fory.serializer.collection.CollectionSerializer; import org.apache.fory.serializer.collection.CollectionSerializers.ArrayListSerializer; +import org.apache.fory.serializer.collection.CollectionSerializers.ConcurrentSkipListSetSerializer; +import org.apache.fory.serializer.collection.CollectionSerializers.CopyOnWriteArrayListSerializer; +import org.apache.fory.serializer.collection.CollectionSerializers.CopyOnWriteArraySetSerializer; import org.apache.fory.serializer.collection.CollectionSerializers.HashSetSerializer; +import org.apache.fory.serializer.collection.CollectionSerializers.LinkedHashSetSerializer; +import org.apache.fory.serializer.collection.CollectionSerializers.SortedSetSerializer; import org.apache.fory.serializer.collection.CollectionSerializers.XlangListDefaultSerializer; import org.apache.fory.serializer.collection.CollectionSerializers.XlangSetDefaultSerializer; import org.apache.fory.serializer.collection.MapLikeSerializer; import org.apache.fory.serializer.collection.MapSerializer; +import org.apache.fory.serializer.collection.MapSerializers.ConcurrentHashMapSerializer; +import org.apache.fory.serializer.collection.MapSerializers.ConcurrentSkipListMapSerializer; +import org.apache.fory.serializer.collection.MapSerializers.LinkedHashMapSerializer; +import org.apache.fory.serializer.collection.MapSerializers.SortedMapSerializer; import org.apache.fory.serializer.collection.MapSerializers.XlangMapSerializer; import org.apache.fory.type.BFloat16; import org.apache.fory.type.BFloat16Array; @@ -1100,6 +1117,11 @@ private void registerDefaultTypes() { new PrimitiveArraySerializers.DoubleArraySerializer(this)); // Collections registerType(Types.LIST, ArrayList.class, new ArrayListSerializer(this)); + registerType(Types.LIST, LinkedList.class, new CollectionSerializer(this, LinkedList.class)); + registerType( + Types.LIST, + CopyOnWriteArrayList.class, + new CopyOnWriteArrayListSerializer(this, CopyOnWriteArrayList.class)); registerType( Types.LIST, Object[].class, @@ -1127,22 +1149,30 @@ private void registerDefaultTypes() { // Sets registerType(Types.SET, HashSet.class, new HashSetSerializer(this)); + registerType(Types.SET, LinkedHashSet.class, new LinkedHashSetSerializer(this)); + registerType(Types.SET, TreeSet.class, new SortedSetSerializer<>(this, TreeSet.class)); + registerType( + Types.SET, + ConcurrentSkipListSet.class, + new ConcurrentSkipListSetSerializer(this, ConcurrentSkipListSet.class)); registerType( Types.SET, - LinkedHashSet.class, - new org.apache.fory.serializer.collection.CollectionSerializers.LinkedHashSetSerializer( - this)); + CopyOnWriteArraySet.class, + new CopyOnWriteArraySetSerializer(this, CopyOnWriteArraySet.class)); registerType(Types.SET, Set.class, new XlangSetDefaultSerializer(this, Set.class)); // Maps + registerType(Types.MAP, HashMap.class, new HashMapSerializer(this)); + registerType(Types.MAP, LinkedHashMap.class, new LinkedHashMapSerializer(this)); + registerType(Types.MAP, TreeMap.class, new SortedMapSerializer<>(this, TreeMap.class)); registerType( Types.MAP, - HashMap.class, - new org.apache.fory.serializer.collection.MapSerializers.HashMapSerializer(this)); + ConcurrentHashMap.class, + new ConcurrentHashMapSerializer(this, ConcurrentHashMap.class)); registerType( Types.MAP, - LinkedHashMap.class, - new org.apache.fory.serializer.collection.MapSerializers.LinkedHashMapSerializer(this)); + ConcurrentSkipListMap.class, + new ConcurrentSkipListMapSerializer(this, ConcurrentSkipListMap.class)); registerType(Types.MAP, Map.class, new XlangMapSerializer(this, Map.class)); registerUnionTypes(); diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectStreamSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectStreamSerializer.java index c3865b2f85..31c0b3c488 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectStreamSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectStreamSerializer.java @@ -686,8 +686,6 @@ private static class StreamTypeInfo { private final Consumer readObjectNoDataFunc; private StreamTypeInfo(Class type) { - // _JDKAccess owns version-specific discovery of serialization hooks without requiring - // ObjectStreamClass private-field access or a java.io package open. Method writeMethod = null; Method readMethod = null; Method noDataMethod = null; @@ -696,9 +694,9 @@ private StreamTypeInfo(Class type) { readMethod = JavaSerializer.getReadRefMethod(type, false); noDataMethod = JavaSerializer.getReadRefNoData(type, false); } else { - writeMethod = _JDKAccess.getSerializationWriteObjectMethod(type); - readMethod = _JDKAccess.getSerializationReadObjectMethod(type); - noDataMethod = _JDKAccess.getSerializationReadObjectNoDataMethod(type); + writeMethod = SerializationHookLookup.getWriteObjectMethod(type); + readMethod = SerializationHookLookup.getReadObjectMethod(type); + noDataMethod = SerializationHookLookup.getReadObjectNoDataMethod(type); if (writeMethod == null) { writeMethod = JavaSerializer.getWriteObjectMethod(type, false); } @@ -715,7 +713,7 @@ private StreamTypeInfo(Class type) { this.defaultReadObjectHandle = AndroidSupport.IS_ANDROID || GraalvmSupport.IN_GRAALVM_NATIVE_IMAGE ? null - : _JDKAccess.getSerializationDefaultReadObjectHandle(type); + : SerializationHookLookup.getDefaultReadObjectHandle(type); if (AndroidSupport.IS_ANDROID) { makeAccessible(writeObjectMethod); makeAccessible(readObjectMethod); diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/PlatformStringUtils.java b/java/fory-core/src/main/java/org/apache/fory/serializer/PlatformStringUtils.java index 57e2e434e6..3614e817be 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/PlatformStringUtils.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/PlatformStringUtils.java @@ -19,17 +19,19 @@ package org.apache.fory.serializer; -import java.nio.charset.StandardCharsets; +import java.lang.reflect.Field; import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.memory.NativeByteOrder; import org.apache.fory.platform.AndroidSupport; import org.apache.fory.platform.GraalvmSupport; import org.apache.fory.platform.internal._JDKAccess; +import org.apache.fory.platform.internal._UnsafeUtils; +import org.apache.fory.util.Preconditions; import sun.misc.Unsafe; /** Platform-owned string internals used by {@link StringSerializer}. */ final class PlatformStringUtils { - private static final Unsafe UNSAFE = AndroidSupport.IS_ANDROID ? null : _JDKAccess.UNSAFE; + private static final Unsafe UNSAFE = AndroidSupport.IS_ANDROID ? null : _UnsafeUtils.UNSAFE; private static final int BYTE_ARRAY_OFFSET; private static final int CHAR_ARRAY_OFFSET; @@ -45,31 +47,117 @@ final class PlatformStringUtils { } } - static final boolean JDK_STRING_FIELD_ACCESS = - !AndroidSupport.IS_ANDROID - && !GraalvmSupport.IN_GRAALVM_NATIVE_IMAGE - && _JDKAccess.JDK_STRING_FIELD_ACCESS; + private static final StringFields STRING_FIELDS = stringFields(); + + static final boolean JDK_STRING_FIELD_ACCESS = STRING_FIELDS.fieldAccess; static final boolean STRING_VALUE_FIELD_IS_CHARS = - JDK_STRING_FIELD_ACCESS && _JDKAccess.STRING_VALUE_FIELD_IS_CHARS; + JDK_STRING_FIELD_ACCESS && STRING_FIELDS.valueFieldIsChars; static final boolean STRING_VALUE_FIELD_IS_BYTES = - JDK_STRING_FIELD_ACCESS && _JDKAccess.STRING_VALUE_FIELD_IS_BYTES; - static final boolean STRING_HAS_COUNT_OFFSET = - JDK_STRING_FIELD_ACCESS && _JDKAccess.STRING_HAS_COUNT_OFFSET; - - private static final long STRING_VALUE_FIELD_OFFSET = - JDK_STRING_FIELD_ACCESS ? _JDKAccess.STRING_VALUE_FIELD_OFFSET : -1; - private static final long STRING_CODER_FIELD_OFFSET = - JDK_STRING_FIELD_ACCESS ? _JDKAccess.STRING_CODER_FIELD_OFFSET : -1; - private static final long STRING_COUNT_FIELD_OFFSET = - JDK_STRING_FIELD_ACCESS ? _JDKAccess.STRING_COUNT_FIELD_OFFSET : -1; - private static final long STRING_OFFSET_FIELD_OFFSET = - JDK_STRING_FIELD_ACCESS ? _JDKAccess.STRING_OFFSET_FIELD_OFFSET : -1; - - private static final byte LATIN1 = 0; - private static final byte UTF16 = 1; + JDK_STRING_FIELD_ACCESS && STRING_FIELDS.valueFieldIsBytes; + static final boolean STRING_HAS_COUNT_OFFSET = JDK_STRING_FIELD_ACCESS && STRING_FIELDS.counted; + + private static final long STRING_VALUE_FIELD_OFFSET = STRING_FIELDS.valueOffset; + private static final long STRING_CODER_FIELD_OFFSET = STRING_FIELDS.coderOffset; + private static final long STRING_COUNT_FIELD_OFFSET = STRING_FIELDS.countOffset; + private static final long STRING_OFFSET_FIELD_OFFSET = STRING_FIELDS.offsetOffset; private PlatformStringUtils() {} + private static StringFields stringFields() { + if (AndroidSupport.IS_ANDROID + || GraalvmSupport.IN_GRAALVM_NATIVE_IMAGE + || !_JDKAccess.JDK_INTERNAL_FIELD_ACCESS) { + return StringFields.noAccess(); + } + try { + Field valueField = String.class.getDeclaredField("value"); + boolean valueFieldIsChars = valueField.getType() == char[].class; + boolean valueFieldIsBytes = valueField.getType() == byte[].class; + long valueOffset = UNSAFE.objectFieldOffset(valueField); + Field countField = getStringFieldNullable("count"); + Field offsetField = getStringFieldNullable("offset"); + boolean counted = false; + long countOffset = -1; + long offsetOffset = -1; + if (countField != null || offsetField != null) { + Preconditions.checkArgument( + countField != null && offsetField != null, "Current jdk not supported"); + Preconditions.checkArgument( + countField.getType() == int.class && offsetField.getType() == int.class, + "Current jdk not supported"); + counted = true; + countOffset = UNSAFE.objectFieldOffset(countField); + offsetOffset = UNSAFE.objectFieldOffset(offsetField); + } + long coderOffset = valueFieldIsBytes ? StringCoderField.OFFSET : -1; + return new StringFields( + true, + valueFieldIsChars, + valueFieldIsBytes, + counted, + valueOffset, + coderOffset, + countOffset, + offsetOffset); + } catch (NoSuchFieldException e) { + throw new RuntimeException(e); + } + } + + private static Field getStringFieldNullable(String fieldName) { + try { + return String.class.getDeclaredField(fieldName); + } catch (NoSuchFieldException e) { + return null; + } + } + + private static class StringCoderField { + private static final long OFFSET; + + static { + try { + OFFSET = UNSAFE.objectFieldOffset(String.class.getDeclaredField("coder")); + } catch (NoSuchFieldException e) { + throw new RuntimeException(e); + } + } + } + + private static final class StringFields { + private final boolean fieldAccess; + private final boolean valueFieldIsChars; + private final boolean valueFieldIsBytes; + private final boolean counted; + private final long valueOffset; + private final long coderOffset; + private final long countOffset; + private final long offsetOffset; + + private StringFields( + boolean fieldAccess, + boolean valueFieldIsChars, + boolean valueFieldIsBytes, + boolean counted, + long valueOffset, + long coderOffset, + long countOffset, + long offsetOffset) { + this.fieldAccess = fieldAccess; + this.valueFieldIsChars = valueFieldIsChars; + this.valueFieldIsBytes = valueFieldIsBytes; + this.counted = counted; + this.valueOffset = valueOffset; + this.coderOffset = coderOffset; + this.countOffset = countOffset; + this.offsetOffset = offsetOffset; + } + + private static StringFields noAccess() { + return new StringFields(false, false, false, false, -1, -1, -1, -1); + } + } + static Object getStringValue(String value) { return UNSAFE.getObject(value, STRING_VALUE_FIELD_OFFSET); } @@ -86,34 +174,6 @@ static int getStringCount(String value) { return UNSAFE.getInt(value, STRING_COUNT_FIELD_OFFSET); } - static String newCharsStringZeroCopy(char[] data) { - if (!JDK_STRING_FIELD_ACCESS) { - return new String(data); - } - return _JDKAccess.newCharsStringZeroCopy(data); - } - - static String newBytesStringZeroCopy(byte coder, byte[] data) { - if (!JDK_STRING_FIELD_ACCESS) { - return newBytesStringSlow(coder, data); - } - return _JDKAccess.newBytesStringZeroCopy(coder, data); - } - - private static String newBytesStringSlow(byte coder, byte[] data) { - if (coder == LATIN1) { - return new String(data, StandardCharsets.ISO_8859_1); - } else if (coder == UTF16) { - char[] chars = new char[data.length >> 1]; - for (int i = 0, j = 0; i < data.length; i += 2) { - chars[j++] = (char) ((data[i] & 0xff) | ((data[i + 1] & 0xff) << 8)); - } - return new String(chars); - } else { - return new String(data, StandardCharsets.UTF_8); - } - } - static long getCharsLong(char[] chars, int charIndex) { if (AndroidSupport.IS_ANDROID) { long c0 = chars[charIndex]; diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/ReplaceResolveSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/ReplaceResolveSerializer.java index 22431bdbbf..ea1a0bcfb8 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/ReplaceResolveSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/ReplaceResolveSerializer.java @@ -76,9 +76,9 @@ private ReplaceResolveInfo(Class cls) { writeReplaceMethod = JavaSerializer.getWriteReplaceMethod(cls); readResolveMethod = JavaSerializer.getReadResolveMethod(cls); } else if (Serializable.class.isAssignableFrom(cls)) { - if (_JDKAccess.isSerializationHookLookupAvailable()) { - writeReplaceMethod = _JDKAccess.getSerializationWriteReplaceMethod(cls); - readResolveMethod = _JDKAccess.getSerializationReadResolveMethod(cls); + if (SerializationHookLookup.isAvailable()) { + writeReplaceMethod = SerializationHookLookup.getWriteReplaceMethod(cls); + readResolveMethod = SerializationHookLookup.getReadResolveMethod(cls); } else { writeReplaceMethod = JavaSerializer.getWriteReplaceMethod(cls); readResolveMethod = JavaSerializer.getReadResolveMethod(cls); diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/SerializationHookLookup.java b/java/fory-core/src/main/java/org/apache/fory/serializer/SerializationHookLookup.java new file mode 100644 index 0000000000..d6e4efaa09 --- /dev/null +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/SerializationHookLookup.java @@ -0,0 +1,131 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.serializer; + +import java.lang.invoke.MethodHandle; +import java.lang.invoke.MethodHandles; +import java.lang.reflect.Method; +import org.apache.fory.platform.internal._JDKAccess; +import org.apache.fory.util.ExceptionUtils; + +final class SerializationHookLookup { + private SerializationHookLookup() {} + + static MethodHandle readResolveHandle(Class type, Method method) + throws IllegalAccessException { + return _JDKAccess._trustedLookup(type).unreflect(method); + } + + private static final class Methods { + private static final Object REFLECTION_FACTORY; + private static final Method WRITE_OBJECT; + private static final Method READ_OBJECT; + private static final Method READ_OBJECT_NO_DATA; + private static final Method DEFAULT_READ_OBJECT; + private static final Method WRITE_REPLACE; + private static final Method READ_RESOLVE; + + static { + Object reflectionFactory = null; + Method writeObject = null; + Method readObject = null; + Method readObjectNoData = null; + Method defaultReadObject = null; + Method writeReplace = null; + Method readResolve = null; + try { + Class factoryClass = Class.forName("sun.reflect.ReflectionFactory"); + Method getReflectionFactory = factoryClass.getDeclaredMethod("getReflectionFactory"); + reflectionFactory = getReflectionFactory.invoke(null); + writeObject = factoryClass.getDeclaredMethod("writeObjectForSerialization", Class.class); + readObject = factoryClass.getDeclaredMethod("readObjectForSerialization", Class.class); + readObjectNoData = + factoryClass.getDeclaredMethod("readObjectNoDataForSerialization", Class.class); + try { + defaultReadObject = + factoryClass.getDeclaredMethod("defaultReadObjectForSerialization", Class.class); + } catch (NoSuchMethodException e) { + ExceptionUtils.ignore(e); + } + writeReplace = factoryClass.getDeclaredMethod("writeReplaceForSerialization", Class.class); + readResolve = factoryClass.getDeclaredMethod("readResolveForSerialization", Class.class); + } catch (Throwable e) { + ExceptionUtils.ignore(e); + } + REFLECTION_FACTORY = reflectionFactory; + WRITE_OBJECT = writeObject; + READ_OBJECT = readObject; + READ_OBJECT_NO_DATA = readObjectNoData; + DEFAULT_READ_OBJECT = defaultReadObject; + WRITE_REPLACE = writeReplace; + READ_RESOLVE = readResolve; + } + } + + private static MethodHandle getHandle(Class type, Method factoryMethod) { + if (Methods.REFLECTION_FACTORY == null || factoryMethod == null) { + return null; + } + try { + return (MethodHandle) factoryMethod.invoke(Methods.REFLECTION_FACTORY, type); + } catch (Throwable e) { + ExceptionUtils.ignore(e); + return null; + } + } + + private static Method getMethod(Class type, Method factoryMethod) { + MethodHandle handle = getHandle(type, factoryMethod); + return handle == null ? null : MethodHandles.reflectAs(Method.class, handle); + } + + static Method getWriteObjectMethod(Class type) { + return getMethod(type, Methods.WRITE_OBJECT); + } + + static Method getReadObjectMethod(Class type) { + return getMethod(type, Methods.READ_OBJECT); + } + + static Method getReadObjectNoDataMethod(Class type) { + return getMethod(type, Methods.READ_OBJECT_NO_DATA); + } + + static MethodHandle getDefaultReadObjectHandle(Class type) { + return getHandle(type, Methods.DEFAULT_READ_OBJECT); + } + + static Method getWriteReplaceMethod(Class type) { + return getMethod(type, Methods.WRITE_REPLACE); + } + + static Method getReadResolveMethod(Class type) { + return getMethod(type, Methods.READ_RESOLVE); + } + + static boolean isAvailable() { + return Methods.REFLECTION_FACTORY != null + && Methods.WRITE_OBJECT != null + && Methods.READ_OBJECT != null + && Methods.READ_OBJECT_NO_DATA != null + && Methods.WRITE_REPLACE != null + && Methods.READ_RESOLVE != null; + } +} diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/SerializedLambdaSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/SerializedLambdaSerializer.java index f06b5cb548..0c225bddaf 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/SerializedLambdaSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/SerializedLambdaSerializer.java @@ -29,7 +29,6 @@ import org.apache.fory.exception.ForyException; import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.platform.AndroidSupport; -import org.apache.fory.platform.internal._JDKAccess; import org.apache.fory.resolver.TypeResolver; import org.apache.fory.util.Preconditions; @@ -53,7 +52,8 @@ public class SerializedLambdaSerializer extends Serializer { Method readResolveMethod = JavaSerializer.getReadResolveMethod(SERIALIZED_LAMBDA); Preconditions.checkNotNull( readResolveMethod, "Missing readResolve for " + SERIALIZED_LAMBDA); - READ_RESOLVE_HANDLE = _JDKAccess.readResolveHandle(SERIALIZED_LAMBDA, readResolveMethod); + READ_RESOLVE_HANDLE = + SerializationHookLookup.readResolveHandle(SERIALIZED_LAMBDA, readResolveMethod); } catch (IllegalAccessException e) { throw new ForyException(e); } diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/StringSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/StringSerializer.java index 8a2d994ccb..f0f956d84a 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/StringSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/StringSerializer.java @@ -23,8 +23,15 @@ import static org.apache.fory.util.StringUtils.MULTI_CHARS_NON_ASCII_MASK; import static org.apache.fory.util.StringUtils.MULTI_CHARS_NON_LATIN_MASK; +import java.lang.invoke.CallSite; +import java.lang.invoke.LambdaMetafactory; +import java.lang.invoke.MethodHandle; +import java.lang.invoke.MethodHandles.Lookup; +import java.lang.invoke.MethodType; import java.nio.charset.StandardCharsets; import java.util.Arrays; +import java.util.function.BiFunction; +import java.util.function.Function; import org.apache.fory.annotation.CodegenInvoke; import org.apache.fory.codegen.Expression; import org.apache.fory.codegen.Expression.Invoke; @@ -37,6 +44,7 @@ import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.memory.NativeByteOrder; import org.apache.fory.platform.AndroidSupport; +import org.apache.fory.platform.internal._JDKAccess; import org.apache.fory.util.MathUtils; import org.apache.fory.util.Preconditions; @@ -57,12 +65,22 @@ public final class StringSerializer extends ImmutableSerializer { PlatformStringUtils.JDK_STRING_FIELD_ACCESS; private static final byte LATIN1 = 0; + private static final Byte LATIN1_BOXED = LATIN1; private static final byte UTF16 = 1; + private static final Byte UTF16_BOXED = UTF16; private static final byte UTF8 = 2; private static final int DEFAULT_BUFFER_SIZE = 1024; private static final boolean STRING_HAS_COUNT_OFFSET = PlatformStringUtils.STRING_HAS_COUNT_OFFSET; + private static final Lookup STRING_LOOKUP = + JDK_INTERNAL_FIELD_ACCESS ? _JDKAccess._trustedLookup(String.class) : null; + private static final BiFunction CHARS_STRING_ZERO_COPY_CTR = + JDK_INTERNAL_FIELD_ACCESS ? getCharsStringZeroCopyCtr() : null; + private static final BiFunction BYTES_STRING_ZERO_COPY_CTR = + JDK_INTERNAL_FIELD_ACCESS ? getBytesStringZeroCopyCtr() : null; + private static final Function LATIN_BYTES_STRING_ZERO_COPY_CTR = + JDK_INTERNAL_FIELD_ACCESS ? getLatinBytesStringZeroCopyCtr() : null; private final boolean compressString; private final boolean writeNumUtf16BytesForUtf8Encoding; @@ -874,13 +892,126 @@ private void writeBytesUTF8PerfOptimized(MemoryBuffer buffer, byte[] bytes) { } public static String newCharsStringZeroCopy(char[] data) { - return PlatformStringUtils.newCharsStringZeroCopy(data); + if (!JDK_INTERNAL_FIELD_ACCESS) { + return new String(data); + } + if (!STRING_VALUE_FIELD_IS_CHARS) { + throw new IllegalStateException("String value isn't char[], current java isn't supported"); + } + return CHARS_STRING_ZERO_COPY_CTR.apply(data, Boolean.TRUE); } // coder param first to make inline call args // `(buffer.readByte(), buffer.readBytesWithSizeEmbedded())` work. public static String newBytesStringZeroCopy(byte coder, byte[] data) { - return PlatformStringUtils.newBytesStringZeroCopy(coder, data); + if (!JDK_INTERNAL_FIELD_ACCESS) { + return newBytesStringSlow(coder, data); + } + if (coder == LATIN1) { + if (LATIN_BYTES_STRING_ZERO_COPY_CTR != null) { + return LATIN_BYTES_STRING_ZERO_COPY_CTR.apply(data); + } + return BYTES_STRING_ZERO_COPY_CTR.apply(data, LATIN1_BOXED); + } else if (coder == UTF16) { + return BYTES_STRING_ZERO_COPY_CTR.apply(data, UTF16_BOXED); + } else { + return BYTES_STRING_ZERO_COPY_CTR.apply(data, coder); + } + } + + private static String newBytesStringSlow(byte coder, byte[] data) { + if (coder == LATIN1) { + return new String(data, StandardCharsets.ISO_8859_1); + } else if (coder == UTF16) { + char[] chars = new char[data.length >> 1]; + for (int i = 0, j = 0; i < data.length; i += 2) { + chars[j++] = (char) ((data[i] & 0xff) | ((data[i + 1] & 0xff) << 8)); + } + return new String(chars); + } else { + return new String(data, StandardCharsets.UTF_8); + } + } + + private static BiFunction getCharsStringZeroCopyCtr() { + if (!STRING_VALUE_FIELD_IS_CHARS) { + return null; + } + MethodHandle handle = getJavaStringZeroCopyCtrHandle(); + if (handle == null) { + return null; + } + try { + CallSite callSite = + LambdaMetafactory.metafactory( + STRING_LOOKUP, + "apply", + MethodType.methodType(BiFunction.class), + handle.type().generic(), + handle, + handle.type()); + return (BiFunction) callSite.getTarget().invokeExact(); + } catch (Throwable e) { + return null; + } + } + + private static BiFunction getBytesStringZeroCopyCtr() { + if (!STRING_VALUE_FIELD_IS_BYTES) { + return null; + } + MethodHandle handle = getJavaStringZeroCopyCtrHandle(); + if (handle == null) { + return null; + } + try { + MethodType instantiatedMethodType = + MethodType.methodType(handle.type().returnType(), new Class[] {byte[].class, Byte.class}); + CallSite callSite = + LambdaMetafactory.metafactory( + STRING_LOOKUP, + "apply", + MethodType.methodType(BiFunction.class), + handle.type().generic(), + handle, + instantiatedMethodType); + return (BiFunction) callSite.getTarget().invokeExact(); + } catch (Throwable e) { + return null; + } + } + + private static Function getLatinBytesStringZeroCopyCtr() { + if (!STRING_VALUE_FIELD_IS_BYTES || STRING_LOOKUP == null) { + return null; + } + try { + Class clazz = Class.forName("java.lang.StringCoding"); + Lookup caller = STRING_LOOKUP.in(clazz); + MethodHandle handle = + caller.findStatic( + clazz, "newStringLatin1", MethodType.methodType(String.class, byte[].class)); + return _JDKAccess.makeFunction(caller, handle, Function.class); + } catch (Throwable e) { + return null; + } + } + + private static MethodHandle getJavaStringZeroCopyCtrHandle() { + if (STRING_LOOKUP == null) { + return null; + } + try { + if (STRING_VALUE_FIELD_IS_CHARS) { + return STRING_LOOKUP.findConstructor( + String.class, MethodType.methodType(void.class, char[].class, boolean.class)); + } else { + return STRING_LOOKUP.findConstructor( + String.class, MethodType.methodType(void.class, byte[].class, byte.class)); + } + } catch (Exception e) { + return null; + } } private static void writeCharsUTF16BEToHeap( diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/CollectionSerializers.java b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/CollectionSerializers.java index db5dcf80f0..72758e6598 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/CollectionSerializers.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/CollectionSerializers.java @@ -77,6 +77,18 @@ */ @SuppressWarnings({"unchecked", "rawtypes"}) public class CollectionSerializers { + private static final Comparator NATURAL_ORDER_COMPARATOR = Comparator.naturalOrder(); + + private static void requireXlangNaturalOrdering(Class type, Comparator comparator) { + if (comparator != null && comparator != NATURAL_ORDER_COMPARATOR) { + throw new UnsupportedOperationException( + "Xlang serialization of " + + type.getName() + + " with a custom comparator is unsupported because the xlang set wire format " + + "does not encode comparators"); + } + } + private static void throwBinarySizeLimitExceeded(long size, int maxBinarySize) { throw new DeserializationException( "Binary payload size " + size + " exceeds max binary size " + maxBinarySize); @@ -245,6 +257,9 @@ public SortedSetSerializer(TypeResolver typeResolver, Class cls) { @Override public Collection onCollectionWrite(WriteContext writeContext, T value) { + if (config.isXlang()) { + requireXlangNaturalOrdering(type, value.comparator()); + } MemoryBuffer buffer = writeContext.getBuffer(); buffer.writeVarUInt32Small7(value.size()); if (config.isXlang()) { @@ -258,12 +273,11 @@ public Collection onCollectionWrite(WriteContext writeContext, T value) { @SuppressWarnings("unchecked") @Override public T newCollection(ReadContext readContext) { - assert !config.isXlang(); MemoryBuffer buffer = readContext.getBuffer(); int numElements = readCollectionSize(buffer); setNumElements(numElements); T collection; - Comparator comparator = (Comparator) readContext.readRef(); + Comparator comparator = config.isXlang() ? null : (Comparator) readContext.readRef(); if (type == TreeSet.class) { collection = (T) new TreeSet(comparator); } else { @@ -517,6 +531,9 @@ public ConcurrentSkipListSetSerializer( @Override public CollectionSnapshot onCollectionWrite( WriteContext writeContext, ConcurrentSkipListSet value) { + if (config.isXlang()) { + requireXlangNaturalOrdering(type, value.comparator()); + } CollectionSnapshot snapshot = super.onCollectionWrite(writeContext, value); if (config.isXlang()) { return snapshot; @@ -530,7 +547,11 @@ public ConcurrentSkipListSet newCollection(ReadContext readContext) { MemoryBuffer buffer = readContext.getBuffer(); int numElements = readCollectionSize(buffer); setNumElements(numElements); - assert !config.isXlang(); + if (config.isXlang()) { + ConcurrentSkipListSet skipListSet = new ConcurrentSkipListSet(); + readContext.reference(skipListSet); + return skipListSet; + } int refId = readContext.lastPreservedRefId(); // It's possible that comparator/elements has circular ref to set. Comparator comparator = (Comparator) readContext.readRef(); diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/MapSerializers.java b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/MapSerializers.java index 43a464b12f..ceaab81a3d 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/MapSerializers.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/MapSerializers.java @@ -65,6 +65,17 @@ */ @SuppressWarnings({"unchecked", "rawtypes"}) public class MapSerializers { + private static final Comparator NATURAL_ORDER_COMPARATOR = Comparator.naturalOrder(); + + private static void requireXlangNaturalOrdering(Class type, Comparator comparator) { + if (comparator != null && comparator != NATURAL_ORDER_COMPARATOR) { + throw new UnsupportedOperationException( + "Xlang serialization of " + + type.getName() + + " with a custom comparator is unsupported because the xlang map wire format " + + "does not encode comparators"); + } + } public static final class HashMapSerializer extends MapSerializer { public HashMapSerializer(TypeResolver typeResolver) { @@ -153,6 +164,9 @@ public SortedMapSerializer(TypeResolver typeResolver, Class cls) { @Override public Map onMapWrite(WriteContext writeContext, T value) { + if (config.isXlang()) { + requireXlangNaturalOrdering(type, value.comparator()); + } MemoryBuffer buffer = writeContext.getBuffer(); buffer.writeVarUInt32Small7(value.size()); if (config.isXlang()) { @@ -166,11 +180,10 @@ public Map onMapWrite(WriteContext writeContext, T value) { @SuppressWarnings("unchecked") @Override public Map newMap(ReadContext readContext) { - assert !config.isXlang(); MemoryBuffer buffer = readContext.getBuffer(); setNumElements(readMapSize(buffer)); T map; - Comparator comparator = (Comparator) readContext.readRef(); + Comparator comparator = config.isXlang() ? null : (Comparator) readContext.readRef(); if (type == TreeMap.class) { map = (T) new TreeMap(comparator); } else { @@ -312,6 +325,9 @@ public ConcurrentSkipListMapSerializer( @Override public MapSnapshot onMapWrite(WriteContext writeContext, ConcurrentSkipListMap value) { + if (config.isXlang()) { + requireXlangNaturalOrdering(type, value.comparator()); + } MapSnapshot snapshot = super.onMapWrite(writeContext, value); if (config.isXlang()) { return snapshot; @@ -325,7 +341,7 @@ public ConcurrentSkipListMap newMap(ReadContext readContext) { MemoryBuffer buffer = readContext.getBuffer(); int numElements = readMapSize(buffer); setNumElements(numElements); - Comparator comparator = (Comparator) readContext.readRef(); + Comparator comparator = config.isXlang() ? null : (Comparator) readContext.readRef(); ConcurrentSkipListMap map = new ConcurrentSkipListMap(comparator); readContext.reference(map); return map; diff --git a/java/fory-core/src/main/java25/org/apache/fory/memory/MemoryBuffer.java b/java/fory-core/src/main/java25/org/apache/fory/memory/MemoryBuffer.java index ecd6fb1745..8472bfd3f0 100644 --- a/java/fory-core/src/main/java25/org/apache/fory/memory/MemoryBuffer.java +++ b/java/fory-core/src/main/java25/org/apache/fory/memory/MemoryBuffer.java @@ -30,7 +30,6 @@ import org.apache.fory.annotation.CodegenInvoke; import org.apache.fory.io.AbstractStreamReader; import org.apache.fory.io.ForyStreamReader; -import org.apache.fory.platform.AndroidSupport; /** * A class for operations on memory managed by Fory. The buffer may be backed by heap memory (byte @@ -220,11 +219,9 @@ private static long getAddress(ByteBuffer buffer) { public void initByteBuffer(ByteBuffer buffer, int size) { if (buffer.isDirect()) { - if (AndroidSupport.IS_ANDROID) { - MemoryOps.throwDirectByteBufferUnsupported(); - } else { + initOffHeapBuffer(0, size, buffer); - } + } else if (buffer.hasArray()) { initHeapBuffer(buffer.array(), buffer.arrayOffset(), size); } else { @@ -264,9 +261,7 @@ public MemoryBuffer getBuffer() { } public void initHeapBuffer(byte[] buffer, int offset, int length) { - if (AndroidSupport.IS_ANDROID) { - MemoryOps.initHeapBuffer(this, buffer, offset, length); - } else { + if (buffer == null) { throw new NullPointerException("buffer"); } @@ -278,7 +273,7 @@ public void initHeapBuffer(byte[] buffer, int offset, int length) { this.address = startPos; this.size = length; this.addressLimit = startPos + length; - } + } private byte loadByte(long pos) { @@ -744,9 +739,7 @@ public void get(int offset, ByteBuffer target, int numBytes) { throw new IllegalArgumentException("read only buffer"); } final int targetPos = target.position(); - if (AndroidSupport.IS_ANDROID) { - MemoryOps.get(this, offset, target, numBytes); - } else if (target.isDirect()) { + if (target.isDirect()) { final long sourceAddr = address + offset; if (sourceAddr <= addressLimit - numBytes) { ByteBuffer duplicate = target.duplicate(); @@ -770,9 +763,7 @@ public void put(int offset, ByteBuffer source, int numBytes) { throwOOBException(); } final int sourcePos = source.position(); - if (AndroidSupport.IS_ANDROID) { - MemoryOps.put(this, offset, source, numBytes); - } else if (source.isDirect()) { + if (source.isDirect()) { final long targetAddr = address + offset; if (targetAddr <= addressLimit - numBytes) { ByteBuffer duplicate = source.duplicate(); @@ -821,9 +812,7 @@ public void put(int index, byte[] src, int offset, int length) { } public byte getByte(int index) { - if (AndroidSupport.IS_ANDROID) { - return MemoryOps.getByte(this, index); - } + final long pos = address + index; checkPosition(index, pos, 1); return loadByte(pos); @@ -832,46 +821,36 @@ public byte getByte(int index) { // CHECKSTYLE.OFF:MethodName public byte _unsafeGetByte(int index) { // CHECKSTYLE.ON:MethodName - if (AndroidSupport.IS_ANDROID) { - return MemoryOps.unsafeGetByte(this, index); - } + return loadByte(address + index); } public void putByte(int index, int b) { - if (AndroidSupport.IS_ANDROID) { - MemoryOps.putByte(this, index, b); - } else { + final long pos = address + index; checkPosition(index, pos, 1); storeByte(pos, (byte) b); - } + } public void putByte(int index, byte b) { - if (AndroidSupport.IS_ANDROID) { - MemoryOps.putByte(this, index, b); - } else { + final long pos = address + index; checkPosition(index, pos, 1); storeByte(pos, b); - } + } // CHECKSTYLE.OFF:MethodName public void _unsafePutByte(int index, byte b) { // CHECKSTYLE.ON:MethodName - if (AndroidSupport.IS_ANDROID) { - MemoryOps.unsafePutByte(this, index, b); - } else { + storeByte(address + index, b); - } + } public boolean getBoolean(int index) { - if (AndroidSupport.IS_ANDROID) { - return MemoryOps.getBoolean(this, index); - } + final long pos = address + index; checkPosition(index, pos, 1); return loadByte(pos) != 0; @@ -880,34 +859,26 @@ public boolean getBoolean(int index) { // CHECKSTYLE.OFF:MethodName public boolean _unsafeGetBoolean(int index) { // CHECKSTYLE.ON:MethodName - if (AndroidSupport.IS_ANDROID) { - return MemoryOps.unsafeGetBoolean(this, index); - } + return loadByte(address + index) != 0; } public void putBoolean(int index, boolean value) { - if (AndroidSupport.IS_ANDROID) { - MemoryOps.putBoolean(this, index, value); - } else { + storeByte(address + index, (value ? (byte) 1 : (byte) 0)); - } + } // CHECKSTYLE.OFF:MethodName public void _unsafePutBoolean(int index, boolean value) { // CHECKSTYLE.ON:MethodName - if (AndroidSupport.IS_ANDROID) { - MemoryOps.putBoolean(this, index, value); - } else { + storeByte(address + index, (value ? (byte) 1 : (byte) 0)); - } + } public char getChar(int index) { - if (AndroidSupport.IS_ANDROID) { - return MemoryOps.getChar(this, index); - } + final long pos = address + index; checkPosition(index, pos, 2); char c = loadChar(pos); @@ -917,43 +888,35 @@ public char getChar(int index) { // CHECKSTYLE.OFF:MethodName public char _unsafeGetChar(int index) { // CHECKSTYLE.ON:MethodName - if (AndroidSupport.IS_ANDROID) { - return MemoryOps.unsafeGetChar(this, index); - } + char c = loadChar(address + index); return LITTLE_ENDIAN ? c : Character.reverseBytes(c); } public void putChar(int index, char value) { - if (AndroidSupport.IS_ANDROID) { - MemoryOps.putChar(this, index, value); - } else { + final long pos = address + index; checkPosition(index, pos, 2); if (!LITTLE_ENDIAN) { value = Character.reverseBytes(value); } storeChar(pos, value); - } + } // CHECKSTYLE.OFF:MethodName public void _unsafePutChar(int index, char value) { // CHECKSTYLE.ON:MethodName - if (AndroidSupport.IS_ANDROID) { - MemoryOps.unsafePutChar(this, index, value); - } else { + if (!LITTLE_ENDIAN) { value = Character.reverseBytes(value); } storeChar(address + index, value); - } + } public short getInt16(int index) { - if (AndroidSupport.IS_ANDROID) { - return MemoryOps.getInt16(this, index); - } + final long pos = address + index; checkPosition(index, pos, 2); short v = loadShort(pos); @@ -961,24 +924,20 @@ public short getInt16(int index) { } public void putInt16(int index, short value) { - if (AndroidSupport.IS_ANDROID) { - MemoryOps.putInt16(this, index, value); - } else { + final long pos = address + index; checkPosition(index, pos, 2); if (!LITTLE_ENDIAN) { value = Short.reverseBytes(value); } storeShort(pos, value); - } + } // CHECKSTYLE.OFF:MethodName public short _unsafeGetInt16(int index) { // CHECKSTYLE.ON:MethodName - if (AndroidSupport.IS_ANDROID) { - return MemoryOps.unsafeGetInt16(this, index); - } + short v = loadShort(address + index); return LITTLE_ENDIAN ? v : Short.reverseBytes(v); } @@ -986,20 +945,16 @@ public short _unsafeGetInt16(int index) { // CHECKSTYLE.OFF:MethodName public void _unsafePutInt16(int index, short value) { // CHECKSTYLE.ON:MethodName - if (AndroidSupport.IS_ANDROID) { - MemoryOps.unsafePutInt16(this, index, value); - } else { + if (!LITTLE_ENDIAN) { value = Short.reverseBytes(value); } storeShort(address + index, value); - } + } public int getInt32(int index) { - if (AndroidSupport.IS_ANDROID) { - return MemoryOps.getInt32(this, index); - } + final long pos = address + index; checkPosition(index, pos, 4); int v = loadInt(pos); @@ -1007,24 +962,20 @@ public int getInt32(int index) { } public void putInt32(int index, int value) { - if (AndroidSupport.IS_ANDROID) { - MemoryOps.putInt32(this, index, value); - } else { + final long pos = address + index; checkPosition(index, pos, 4); if (!LITTLE_ENDIAN) { value = Integer.reverseBytes(value); } storeInt(pos, value); - } + } // CHECKSTYLE.OFF:MethodName public int _unsafeGetInt32(int index) { // CHECKSTYLE.ON:MethodName - if (AndroidSupport.IS_ANDROID) { - return MemoryOps.unsafeGetInt32(this, index); - } + int v = loadInt(address + index); return LITTLE_ENDIAN ? v : Integer.reverseBytes(v); } @@ -1032,20 +983,16 @@ public int _unsafeGetInt32(int index) { // CHECKSTYLE.OFF:MethodName public void _unsafePutInt32(int index, int value) { // CHECKSTYLE.ON:MethodName - if (AndroidSupport.IS_ANDROID) { - MemoryOps.unsafePutInt32(this, index, value); - } else { + if (!LITTLE_ENDIAN) { value = Integer.reverseBytes(value); } storeInt(address + index, value); - } + } public long getInt64(int index) { - if (AndroidSupport.IS_ANDROID) { - return MemoryOps.getInt64(this, index); - } + final long pos = address + index; checkPosition(index, pos, 8); long v = loadLong(pos); @@ -1053,24 +1000,20 @@ public long getInt64(int index) { } public void putInt64(int index, long value) { - if (AndroidSupport.IS_ANDROID) { - MemoryOps.putInt64(this, index, value); - } else { + final long pos = address + index; checkPosition(index, pos, 8); if (!LITTLE_ENDIAN) { value = Long.reverseBytes(value); } storeLong(pos, value); - } + } // CHECKSTYLE.OFF:MethodName public long _unsafeGetInt64(int index) { // CHECKSTYLE.ON:MethodName - if (AndroidSupport.IS_ANDROID) { - return MemoryOps.unsafeGetInt64(this, index); - } + long v = loadLong(address + index); return LITTLE_ENDIAN ? v : Long.reverseBytes(v); } @@ -1078,20 +1021,16 @@ public long _unsafeGetInt64(int index) { // CHECKSTYLE.OFF:MethodName public void _unsafePutInt64(int index, long value) { // CHECKSTYLE.ON:MethodName - if (AndroidSupport.IS_ANDROID) { - MemoryOps.unsafePutInt64(this, index, value); - } else { + if (!LITTLE_ENDIAN) { value = Long.reverseBytes(value); } storeLong(address + index, value); - } + } public float getFloat32(int index) { - if (AndroidSupport.IS_ANDROID) { - return MemoryOps.getFloat32(this, index); - } + final long pos = address + index; checkPosition(index, pos, 4); int v = loadInt(pos); @@ -1102,9 +1041,7 @@ public float getFloat32(int index) { } public void putFloat32(int index, float value) { - if (AndroidSupport.IS_ANDROID) { - MemoryOps.putFloat32(this, index, value); - } else { + final long pos = address + index; checkPosition(index, pos, 4); int v = Float.floatToRawIntBits(value); @@ -1112,13 +1049,11 @@ public void putFloat32(int index, float value) { v = Integer.reverseBytes(v); } storeInt(pos, v); - } + } public double getFloat64(int index) { - if (AndroidSupport.IS_ANDROID) { - return MemoryOps.getFloat64(this, index); - } + final long pos = address + index; checkPosition(index, pos, 8); long v = loadLong(pos); @@ -1129,9 +1064,7 @@ public double getFloat64(int index) { } public void putFloat64(int index, double value) { - if (AndroidSupport.IS_ANDROID) { - MemoryOps.putFloat64(this, index, value); - } else { + final long pos = address + index; checkPosition(index, pos, 8); long v = Double.doubleToRawLongBits(value); @@ -1139,7 +1072,7 @@ public void putFloat64(int index, double value) { v = Long.reverseBytes(v); } storeLong(pos, v); - } + } // Check should be done outside to avoid this method got into the critical path. @@ -1210,30 +1143,26 @@ public void increaseWriterIndex(int diff) { } public void writeBoolean(boolean value) { - if (AndroidSupport.IS_ANDROID) { - MemoryOps.writeBoolean(this, value); - } else { + final int writerIdx = writerIndex; final int newIdx = writerIdx + 1; ensure(newIdx); final long pos = address + writerIdx; storeByte(pos, (byte) (value ? 1 : 0)); writerIndex = newIdx; - } + } // CHECKSTYLE.OFF:MethodName public void _unsafeWriteByte(byte value) { // CHECKSTYLE.ON:MethodName - if (AndroidSupport.IS_ANDROID) { - MemoryOps.unsafeWriteByte(this, value); - } else { + final int writerIdx = writerIndex; final int newIdx = writerIdx + 1; final long pos = address + writerIdx; storeByte(pos, value); writerIndex = newIdx; - } + } public void writeUInt8(int value) { @@ -1241,16 +1170,14 @@ public void writeUInt8(int value) { } public void writeByte(byte value) { - if (AndroidSupport.IS_ANDROID) { - MemoryOps.writeByte(this, value); - } else { + final int writerIdx = writerIndex; final int newIdx = writerIdx + 1; ensure(newIdx); final long pos = address + writerIdx; storeByte(pos, value); writerIndex = newIdx; - } + } public void writeByte(int value) { @@ -1258,9 +1185,7 @@ public void writeByte(int value) { } public void writeChar(char value) { - if (AndroidSupport.IS_ANDROID) { - MemoryOps.writeChar(this, value); - } else { + final int writerIdx = writerIndex; final int newIdx = writerIdx + 2; ensure(newIdx); @@ -1270,13 +1195,11 @@ public void writeChar(char value) { } storeChar(pos, value); writerIndex = newIdx; - } + } public void writeInt16(short value) { - if (AndroidSupport.IS_ANDROID) { - MemoryOps.writeInt16(this, value); - } else { + final int writerIdx = writerIndex; final int newIdx = writerIdx + 2; ensure(newIdx); @@ -1285,13 +1208,11 @@ public void writeInt16(short value) { } storeShort(address + writerIdx, value); writerIndex = newIdx; - } + } public void writeInt32(int value) { - if (AndroidSupport.IS_ANDROID) { - MemoryOps.writeInt32(this, value); - } else { + final int writerIdx = writerIndex; final int newIdx = writerIdx + 4; ensure(newIdx); @@ -1300,13 +1221,11 @@ public void writeInt32(int value) { } storeInt(address + writerIdx, value); writerIndex = newIdx; - } + } public void writeInt64(long value) { - if (AndroidSupport.IS_ANDROID) { - MemoryOps.writeInt64(this, value); - } else { + final int writerIdx = writerIndex; final int newIdx = writerIdx + 8; ensure(newIdx); @@ -1315,13 +1234,11 @@ public void writeInt64(long value) { } storeLong(address + writerIdx, value); writerIndex = newIdx; - } + } public void writeFloat32(float value) { - if (AndroidSupport.IS_ANDROID) { - MemoryOps.writeFloat32(this, value); - } else { + final int writerIdx = writerIndex; final int newIdx = writerIdx + 4; ensure(newIdx); @@ -1331,13 +1248,11 @@ public void writeFloat32(float value) { } storeInt(address + writerIdx, v); writerIndex = newIdx; - } + } public void writeFloat64(double value) { - if (AndroidSupport.IS_ANDROID) { - MemoryOps.writeFloat64(this, value); - } else { + final int writerIdx = writerIndex; final int newIdx = writerIdx + 8; ensure(newIdx); @@ -1347,7 +1262,7 @@ public void writeFloat64(double value) { } storeLong(address + writerIdx, v); writerIndex = newIdx; - } + } /** @@ -1355,9 +1270,7 @@ public void writeFloat64(double value) { * to save one bit. */ public int writeVarInt32(int v) { - if (AndroidSupport.IS_ANDROID) { - return MemoryOps.writeVarInt32(this, v); - } + ensure(writerIndex + 8); // Zigzag encoding: maps negative values to positive values // This works entirely in int without conversion to long @@ -1374,9 +1287,7 @@ public int writeVarInt32(int v) { // CHECKSTYLE.OFF:MethodName public int _unsafeWriteVarInt32(int v) { // CHECKSTYLE.ON:MethodName - if (AndroidSupport.IS_ANDROID) { - return MemoryOps.unsafeWriteVarInt32(this, v); - } + // Zigzag encoding ensures negatives close to zero are encoded in few bytes int varintBytes = _unsafePutVarUInt32(writerIndex, (v << 1) ^ (v >> 31)); writerIndex += varintBytes; @@ -1389,9 +1300,7 @@ public int _unsafeWriteVarInt32(int v) { * @return The number of bytes written. */ public int writeVarUInt32(int v) { - if (AndroidSupport.IS_ANDROID) { - return MemoryOps.writeVarUInt32(this, v); - } + // ensure at least 8 bytes are writable at once, so jvm-jit // generated code is smaller. Otherwise, the reference writer fast path // may be `callee is too large`/`already compiled into a big method` @@ -1408,9 +1317,7 @@ public int writeVarUInt32(int v) { // CHECKSTYLE.OFF:MethodName public int _unsafeWriteVarUInt32(int v) { // CHECKSTYLE.ON:MethodName - if (AndroidSupport.IS_ANDROID) { - return MemoryOps.unsafeWriteVarUInt32(this, v); - } + int varintBytes = _unsafePutVarUInt32(writerIndex, v); writerIndex += varintBytes; return varintBytes; @@ -1421,9 +1328,7 @@ public int _unsafeWriteVarUInt32(int v) { * 127). When the value is equal or greater than 127, the write will be a little slower. */ public int writeVarUInt32Small7(int value) { - if (AndroidSupport.IS_ANDROID) { - return MemoryOps.writeVarUInt32Small7(this, value); - } + ensure(writerIndex + 8); if (value >>> 7 == 0) { storeByte(address + writerIndex++, (byte) value); @@ -1478,9 +1383,7 @@ private int continueWriteVarUInt32Small7(int value) { // CHECKSTYLE.OFF:MethodName public int _unsafePutVarUInt32(int index, int value) { // CHECKSTYLE.ON:MethodName - if (AndroidSupport.IS_ANDROID) { - return MemoryOps.unsafePutVarUInt32(this, index, value); - } + byte[] heap = heapMemory; if (heap != null) { return putVarUInt32Heap(heap, (int) (address + index), value); @@ -1600,9 +1503,7 @@ private int continuePutVarUInt32BigEndian(int index, int encoded, int value) { // CHECKSTYLE.OFF:MethodName public int _unsafePutVarUint36Small(int index, long value) { // CHECKSTYLE.ON:MethodName - if (AndroidSupport.IS_ANDROID) { - return MemoryOps.unsafePutVarUint36Small(this, index, value); - } + long encoded = (value & 0x7F); if (value >>> 7 == 0) { storeByte(address + index, (byte) value); @@ -1673,9 +1574,7 @@ private int continuePutVarUint36SmallBigEndian(int index, long encoded, long val * @return The number of bytes written. */ public int writeVarUInt32Aligned(int value) { - if (AndroidSupport.IS_ANDROID) { - return MemoryOps.writeVarUInt32Aligned(this, value); - } + // Mask first 6 bits, // bit 7 `unset` indicates have next padding bytes, // bit 8 `set` indicates have next data bytes. @@ -1868,9 +1767,7 @@ private int writeVarUInt32Aligned6(int value) { * #writeVarUInt64} to save one bit. */ public int writeVarInt64(long value) { - if (AndroidSupport.IS_ANDROID) { - return MemoryOps.writeVarInt64(this, value); - } + ensure(writerIndex + 9); return _unsafeWriteVarUInt64((value << 1) ^ (value >> 63)); } @@ -1879,16 +1776,12 @@ public int writeVarInt64(long value) { // CHECKSTYLE.OFF:MethodName public int _unsafeWriteVarInt64(long value) { // CHECKSTYLE.ON:MethodName - if (AndroidSupport.IS_ANDROID) { - return MemoryOps.unsafeWriteVarInt64(this, value); - } + return _unsafeWriteVarUInt64((value << 1) ^ (value >> 63)); } public int writeVarUInt64(long value) { - if (AndroidSupport.IS_ANDROID) { - return MemoryOps.writeVarUInt64(this, value); - } + // Var long encoding algorithm is based kryo UnsafeMemoryOutput.writeVarInt64. // var long are written using little endian byte order. ensure(writerIndex + 9); @@ -1899,9 +1792,7 @@ public int writeVarUInt64(long value) { @CodegenInvoke public int _unsafeWriteVarUInt64(long value) { // CHECKSTYLE.ON:MethodName - if (AndroidSupport.IS_ANDROID) { - return MemoryOps.unsafeWriteVarUInt64(this, value); - } + final int writerIndex = this.writerIndex; int varInt; varInt = (int) (value & 0x7F); @@ -1984,9 +1875,7 @@ public int writeTaggedUInt64(long value) { // CHECKSTYLE.OFF:MethodName public int _unsafeWriteTaggedUInt64(long value) { // CHECKSTYLE.ON:MethodName - if (AndroidSupport.IS_ANDROID) { - return MemoryOps.unsafeWriteTaggedUInt64(this, value); - } + final int writerIndex = this.writerIndex; final long pos = address + writerIndex; final byte[] heapMemory = this.heapMemory; @@ -2017,9 +1906,7 @@ public int _unsafeWriteTaggedUInt64(long value) { // CHECKSTYLE.OFF:MethodName public int _unsafeWriteTaggedInt64(long value) { // CHECKSTYLE.ON:MethodName - if (AndroidSupport.IS_ANDROID) { - return MemoryOps.unsafeWriteTaggedInt64(this, value); - } + final int writerIndex = this.writerIndex; final long pos = address + writerIndex; final byte[] heapMemory = this.heapMemory; @@ -2073,21 +1960,17 @@ public void write(ByteBuffer source, int numBytes) { } public void writeBytesWithSize(byte[] values) { - if (AndroidSupport.IS_ANDROID) { - MemoryOps.writeBytesWithSize(this, values); - } else { + writeVarUInt32Small7(values.length); writeBytes(values, 0, values.length); - } + } public void writeBooleansWithSize(boolean[] values) { - if (AndroidSupport.IS_ANDROID) { - MemoryOps.writeBooleansWithSize(this, values); - } else { + writeVarUInt32Small7(values.length); writeBooleans(values, 0, values.length); - } + } public void writeBooleans(boolean[] values) { @@ -2095,25 +1978,21 @@ public void writeBooleans(boolean[] values) { } public void writeBooleans(boolean[] values, int offset, int numElements) { - if (AndroidSupport.IS_ANDROID) { - MemoryOps.writeBooleans(this, values, offset, numElements); - } else { + final int writerIdx = writerIndex; final int newIdx = writerIdx + numElements; ensure(newIdx); writeBooleansFromArray(address + writerIdx, values, BOOLEAN_ARRAY_OFFSET + offset, numElements); writerIndex = newIdx; - } + } public void writeCharsWithSize(char[] values) { - if (AndroidSupport.IS_ANDROID) { - MemoryOps.writeCharsWithSize(this, values); - } else { + int numBytes = Math.multiplyExact(values.length, 2); writeVarUInt32Small7(numBytes); writeChars(values, 0, values.length); - } + } public void writeChars(char[] values) { @@ -2121,26 +2000,22 @@ public void writeChars(char[] values) { } public void writeChars(char[] values, int offset, int numElements) { - if (AndroidSupport.IS_ANDROID) { - MemoryOps.writeChars(this, values, offset, numElements); - } else { + int numBytes = Math.multiplyExact(numElements, 2); final int writerIdx = writerIndex; final int newIdx = writerIdx + numBytes; ensure(newIdx); writeCharsFromArray(address + writerIdx, values, CHAR_ARRAY_OFFSET + offset, numBytes); writerIndex = newIdx; - } + } public void writeShortsWithSize(short[] values) { - if (AndroidSupport.IS_ANDROID) { - MemoryOps.writeShortsWithSize(this, values); - } else { + int numBytes = Math.multiplyExact(values.length, 2); writeVarUInt32Small7(numBytes); writeShorts(values, 0, values.length); - } + } public void writeShorts(short[] values) { @@ -2148,26 +2023,22 @@ public void writeShorts(short[] values) { } public void writeShorts(short[] values, int offset, int numElements) { - if (AndroidSupport.IS_ANDROID) { - MemoryOps.writeShorts(this, values, offset, numElements); - } else { + int numBytes = Math.multiplyExact(numElements, 2); final int writerIdx = writerIndex; final int newIdx = writerIdx + numBytes; ensure(newIdx); writeShortsFromArray(address + writerIdx, values, SHORT_ARRAY_OFFSET + offset, numBytes); writerIndex = newIdx; - } + } public void writeIntsWithSize(int[] values) { - if (AndroidSupport.IS_ANDROID) { - MemoryOps.writeIntsWithSize(this, values); - } else { + int numBytes = Math.multiplyExact(values.length, 4); writeVarUInt32Small7(numBytes); writeInts(values, 0, values.length); - } + } public void writeInts(int[] values) { @@ -2175,26 +2046,22 @@ public void writeInts(int[] values) { } public void writeInts(int[] values, int offset, int numElements) { - if (AndroidSupport.IS_ANDROID) { - MemoryOps.writeInts(this, values, offset, numElements); - } else { + int numBytes = Math.multiplyExact(numElements, 4); final int writerIdx = writerIndex; final int newIdx = writerIdx + numBytes; ensure(newIdx); writeIntsFromArray(address + writerIdx, values, INT_ARRAY_OFFSET + offset, numBytes); writerIndex = newIdx; - } + } public void writeLongsWithSize(long[] values) { - if (AndroidSupport.IS_ANDROID) { - MemoryOps.writeLongsWithSize(this, values); - } else { + int numBytes = Math.multiplyExact(values.length, 8); writeVarUInt32Small7(numBytes); writeLongs(values, 0, values.length); - } + } public void writeLongs(long[] values) { @@ -2202,26 +2069,22 @@ public void writeLongs(long[] values) { } public void writeLongs(long[] values, int offset, int numElements) { - if (AndroidSupport.IS_ANDROID) { - MemoryOps.writeLongs(this, values, offset, numElements); - } else { + int numBytes = Math.multiplyExact(numElements, 8); final int writerIdx = writerIndex; final int newIdx = writerIdx + numBytes; ensure(newIdx); writeLongsFromArray(address + writerIdx, values, LONG_ARRAY_OFFSET + offset, numBytes); writerIndex = newIdx; - } + } public void writeFloatsWithSize(float[] values) { - if (AndroidSupport.IS_ANDROID) { - MemoryOps.writeFloatsWithSize(this, values); - } else { + int numBytes = Math.multiplyExact(values.length, 4); writeVarUInt32Small7(numBytes); writeFloats(values, 0, values.length); - } + } public void writeFloats(float[] values) { @@ -2229,26 +2092,22 @@ public void writeFloats(float[] values) { } public void writeFloats(float[] values, int offset, int numElements) { - if (AndroidSupport.IS_ANDROID) { - MemoryOps.writeFloats(this, values, offset, numElements); - } else { + int numBytes = Math.multiplyExact(numElements, 4); final int writerIdx = writerIndex; final int newIdx = writerIdx + numBytes; ensure(newIdx); writeFloatsFromArray(address + writerIdx, values, FLOAT_ARRAY_OFFSET + offset, numBytes); writerIndex = newIdx; - } + } public void writeDoublesWithSize(double[] values) { - if (AndroidSupport.IS_ANDROID) { - MemoryOps.writeDoublesWithSize(this, values); - } else { + int numBytes = Math.multiplyExact(values.length, 8); writeVarUInt32Small7(numBytes); writeDoubles(values, 0, values.length); - } + } public void writeDoubles(double[] values) { @@ -2256,16 +2115,14 @@ public void writeDoubles(double[] values) { } public void writeDoubles(double[] values, int offset, int numElements) { - if (AndroidSupport.IS_ANDROID) { - MemoryOps.writeDoubles(this, values, offset, numElements); - } else { + int numBytes = Math.multiplyExact(numElements, 8); final int writerIdx = writerIndex; final int newIdx = writerIdx + numBytes; ensure(newIdx); writeDoublesFromArray(address + writerIdx, values, DOUBLE_ARRAY_OFFSET + offset, numBytes); writerIndex = newIdx; - } + } /** For off-heap buffer, this will make a heap buffer internally. */ @@ -2364,9 +2221,7 @@ public int remaining() { } public boolean readBoolean() { - if (AndroidSupport.IS_ANDROID) { - return MemoryOps.readBoolean(this); - } + int readerIdx = readerIndex; // use subtract to avoid overflow if (readerIdx > size - 1) { @@ -2377,9 +2232,7 @@ public boolean readBoolean() { } public int readUInt8() { - if (AndroidSupport.IS_ANDROID) { - return MemoryOps.readUInt8(this); - } + int readerIdx = readerIndex; if (readerIdx > size - 1) { streamReader.fillBuffer(1); @@ -2395,9 +2248,7 @@ public int readUnsignedByte() { } public byte readByte() { - if (AndroidSupport.IS_ANDROID) { - return MemoryOps.readByte(this); - } + int readerIdx = readerIndex; if (readerIdx > size - 1) { streamReader.fillBuffer(1); @@ -2407,9 +2258,7 @@ public byte readByte() { } public char readChar() { - if (AndroidSupport.IS_ANDROID) { - return MemoryOps.readChar(this); - } + int readerIdx = readerIndex; // use subtract to avoid overflow int remaining = size - readerIdx; @@ -2422,9 +2271,7 @@ public char readChar() { } public short readInt16() { - if (AndroidSupport.IS_ANDROID) { - return MemoryOps.readInt16(this); - } + int readerIdx = readerIndex; // use subtract to avoid overflow int remaining = size - readerIdx; @@ -2441,9 +2288,7 @@ public short readInt16() { // CHECKSTYLE.OFF:MethodName public short _readInt16OnLE() { // CHECKSTYLE.ON:MethodName - if (AndroidSupport.IS_ANDROID) { - return MemoryOps.readInt16(this); - } + int readerIdx = readerIndex; // use subtract to avoid overflow int remaining = size - readerIdx; @@ -2459,9 +2304,7 @@ public short _readInt16OnLE() { // CHECKSTYLE.OFF:MethodName public short _readInt16OnBE() { // CHECKSTYLE.ON:MethodName - if (AndroidSupport.IS_ANDROID) { - return MemoryOps.readInt16(this); - } + int readerIdx = readerIndex; // use subtract to avoid overflow int remaining = size - readerIdx; @@ -2473,9 +2316,7 @@ public short _readInt16OnBE() { } public int readInt32() { - if (AndroidSupport.IS_ANDROID) { - return MemoryOps.readInt32(this); - } + int readerIdx = readerIndex; // use subtract to avoid overflow int remaining = size - readerIdx; @@ -2492,9 +2333,7 @@ public int readInt32() { // CHECKSTYLE.OFF:MethodName public int _readInt32OnLE() { // CHECKSTYLE.ON:MethodName - if (AndroidSupport.IS_ANDROID) { - return MemoryOps.readInt32(this); - } + int readerIdx = readerIndex; // use subtract to avoid overflow int remaining = size - readerIdx; @@ -2510,9 +2349,7 @@ public int _readInt32OnLE() { // CHECKSTYLE.OFF:MethodName public int _readInt32OnBE() { // CHECKSTYLE.ON:MethodName - if (AndroidSupport.IS_ANDROID) { - return MemoryOps.readInt32(this); - } + int readerIdx = readerIndex; // use subtract to avoid overflow int remaining = size - readerIdx; @@ -2524,9 +2361,7 @@ public int _readInt32OnBE() { } public long readInt64() { - if (AndroidSupport.IS_ANDROID) { - return MemoryOps.readInt64(this); - } + int readerIdx = readerIndex; // use subtract to avoid overflow int remaining = size - readerIdx; @@ -2543,9 +2378,7 @@ public long readInt64() { // CHECKSTYLE.OFF:MethodName public long _readInt64OnLE() { // CHECKSTYLE.ON:MethodName - if (AndroidSupport.IS_ANDROID) { - return MemoryOps.readInt64(this); - } + int readerIdx = readerIndex; // use subtract to avoid overflow int remaining = size - readerIdx; @@ -2561,9 +2394,7 @@ public long _readInt64OnLE() { // CHECKSTYLE.OFF:MethodName public long _readInt64OnBE() { // CHECKSTYLE.ON:MethodName - if (AndroidSupport.IS_ANDROID) { - return MemoryOps.readInt64(this); - } + int readerIdx = readerIndex; // use subtract to avoid overflow int remaining = size - readerIdx; @@ -2596,9 +2427,7 @@ public long readTaggedUInt64() { // CHECKSTYLE.OFF:MethodName public long _readTaggedUInt64OnLE() { // CHECKSTYLE.ON:MethodName - if (AndroidSupport.IS_ANDROID) { - return MemoryOps.readTaggedUInt64(this); - } + final int readIdx = readerIndex; int diff = size - readIdx; if (diff < 4) { @@ -2621,9 +2450,7 @@ public long _readTaggedUInt64OnLE() { // CHECKSTYLE.OFF:MethodName public long _readTaggedUInt64OnBE() { // CHECKSTYLE.ON:MethodName - if (AndroidSupport.IS_ANDROID) { - return MemoryOps.readTaggedUInt64(this); - } + final int readIdx = readerIndex; int diff = size - readIdx; if (diff < 4) { @@ -2646,9 +2473,7 @@ public long _readTaggedUInt64OnBE() { // CHECKSTYLE.OFF:MethodName public long _readTaggedInt64OnLE() { // CHECKSTYLE.ON:MethodName - if (AndroidSupport.IS_ANDROID) { - return MemoryOps.readTaggedInt64(this); - } + // Duplicate and manual inline for performance. // noinspection Duplicates final int readIdx = readerIndex; @@ -2673,9 +2498,7 @@ public long _readTaggedInt64OnLE() { // CHECKSTYLE.OFF:MethodName public long _readTaggedInt64OnBE() { // CHECKSTYLE.ON:MethodName - if (AndroidSupport.IS_ANDROID) { - return MemoryOps.readTaggedInt64(this); - } + // noinspection Duplicates final int readIdx = readerIndex; int diff = size - readIdx; @@ -2696,9 +2519,7 @@ public long _readTaggedInt64OnBE() { } public float readFloat32() { - if (AndroidSupport.IS_ANDROID) { - return MemoryOps.readFloat32(this); - } + // noinspection Duplicates int readerIdx = readerIndex; // use subtract to avoid overflow @@ -2719,9 +2540,7 @@ public float readFloat32() { // CHECKSTYLE.OFF:MethodName public float _readFloat32OnLE() { // CHECKSTYLE.ON:MethodName - if (AndroidSupport.IS_ANDROID) { - return MemoryOps.readFloat32(this); - } + int readerIdx = readerIndex; // use subtract to avoid overflow int remaining = size - readerIdx; @@ -2737,9 +2556,7 @@ public float _readFloat32OnLE() { // CHECKSTYLE.OFF:MethodName public float _readFloat32OnBE() { // CHECKSTYLE.ON:MethodName - if (AndroidSupport.IS_ANDROID) { - return MemoryOps.readFloat32(this); - } + int readerIdx = readerIndex; // use subtract to avoid overflow int remaining = size - readerIdx; @@ -2752,9 +2569,7 @@ public float _readFloat32OnBE() { } public double readFloat64() { - if (AndroidSupport.IS_ANDROID) { - return MemoryOps.readFloat64(this); - } + // noinspection Duplicates int readerIdx = readerIndex; // use subtract to avoid overflow @@ -2775,9 +2590,7 @@ public double readFloat64() { // CHECKSTYLE.OFF:MethodName public double _readFloat64OnLE() { // CHECKSTYLE.ON:MethodName - if (AndroidSupport.IS_ANDROID) { - return MemoryOps.readFloat64(this); - } + int readerIdx = readerIndex; // use subtract to avoid overflow int remaining = size - readerIdx; @@ -2793,9 +2606,7 @@ public double _readFloat64OnLE() { // CHECKSTYLE.OFF:MethodName public double _readFloat64OnBE() { // CHECKSTYLE.ON:MethodName - if (AndroidSupport.IS_ANDROID) { - return MemoryOps.readFloat64(this); - } + int readerIdx = readerIndex; // use subtract to avoid overflow int remaining = size - readerIdx; @@ -2810,9 +2621,7 @@ public double _readFloat64OnBE() { /** Reads the 1-5 byte int part of a varint. */ @CodegenInvoke public int readVarInt32() { - if (AndroidSupport.IS_ANDROID) { - return MemoryOps.readVarInt32(this); - } + if (LITTLE_ENDIAN) { return _readVarInt32OnLE(); } else { @@ -2825,9 +2634,7 @@ public int readVarInt32() { // CHECKSTYLE.OFF:MethodName public int _readVarInt32OnLE() { // CHECKSTYLE.ON:MethodName - if (AndroidSupport.IS_ANDROID) { - return MemoryOps.readVarInt32(this); - } + // noinspection Duplicates int readIdx = readerIndex; int result; @@ -2875,9 +2682,7 @@ public int _readVarInt32OnLE() { // CHECKSTYLE.OFF:MethodName public int _readVarInt32OnBE() { // CHECKSTYLE.ON:MethodName - if (AndroidSupport.IS_ANDROID) { - return MemoryOps.readVarInt32(this); - } + // noinspection Duplicates int readIdx = readerIndex; int result; @@ -2920,9 +2725,7 @@ public int _readVarInt32OnBE() { } public long readVarUint36Small() { - if (AndroidSupport.IS_ANDROID) { - return MemoryOps.readVarUint36Small(this); - } + // Android exits above. Keep JVM small-varint bulk reads as direct bulk loads instead of calling // `_unsafeGet*` helpers; those helpers carry Android/endian branches and can break inlining. // Duplicate and manual inline for performance. @@ -3032,9 +2835,7 @@ private static void throwMalformedVarUInt32(int fifthByte) { /** Reads the 1-5 byte int part of a non-negative varint. */ public int readVarUInt32() { - if (AndroidSupport.IS_ANDROID) { - return MemoryOps.readVarUInt32(this); - } + int readIdx = readerIndex; if (size - readIdx < 5) { return readVarUInt32Slow(); @@ -3081,9 +2882,7 @@ public int readVarUInt32() { * 127). When the value is equal or greater than 127, the read will be a little slower. */ public int readVarUInt32Small7() { - if (AndroidSupport.IS_ANDROID) { - return MemoryOps.readVarUInt32(this); - } + int readIdx = readerIndex; if (size - readIdx > 0) { byte v = loadByte(address + readIdx++); @@ -3100,9 +2899,7 @@ public int readVarUInt32Small7() { * 16384). When the value is equal or greater than 16384, the read will be a little slower. */ public int readVarUInt32Small14() { - if (AndroidSupport.IS_ANDROID) { - return MemoryOps.readVarUInt32(this); - } + int readIdx = readerIndex; if (size - readIdx >= 5) { int fourByteValue = loadInt(address + readIdx++); @@ -3150,9 +2947,7 @@ private int continueReadVarUInt32(int readIdx, int bulkRead, int value) { /** Reads the 1-9 byte int part of a var long. */ public long readVarInt64() { - if (AndroidSupport.IS_ANDROID) { - return MemoryOps.readVarInt64(this); - } + return LITTLE_ENDIAN ? _readVarInt64OnLE() : _readVarInt64OnBE(); } @@ -3160,9 +2955,7 @@ public long readVarInt64() { // CHECKSTYLE.OFF:MethodName public long _readVarInt64OnLE() { // CHECKSTYLE.ON:MethodName - if (AndroidSupport.IS_ANDROID) { - return MemoryOps.readVarInt64(this); - } + // Duplicate and manual inline for performance. // noinspection Duplicates int readIdx = readerIndex; @@ -3195,9 +2988,7 @@ public long _readVarInt64OnLE() { // CHECKSTYLE.OFF:MethodName public long _readVarInt64OnBE() { // CHECKSTYLE.ON:MethodName - if (AndroidSupport.IS_ANDROID) { - return MemoryOps.readVarInt64(this); - } + int readIdx = readerIndex; long result; if (size - readIdx < 9) { @@ -3226,9 +3017,7 @@ public long _readVarInt64OnBE() { /** Reads the 1-9 byte int part of a non-negative var long. */ public long readVarUInt64() { - if (AndroidSupport.IS_ANDROID) { - return MemoryOps.readVarUInt64(this); - } + int readIdx = readerIndex; if (size - readIdx < 9) { return readVarUInt64Slow(); @@ -3333,9 +3122,7 @@ private long readVarUInt64Slow() { /** Reads the 1-9 byte int part of an aligned varint. */ public int readAlignedVarUInt32() { - if (AndroidSupport.IS_ANDROID) { - return MemoryOps.readAlignedVarUInt32(this); - } + int readerIdx = readerIndex; // use subtract to avoid overflow if (readerIdx < size - 10) { @@ -3462,9 +3249,7 @@ public void readBytes(byte[] dst) { /** Read {@code len} bytes into a long using little-endian order. */ public long readBytesAsInt64(int len) { - if (AndroidSupport.IS_ANDROID) { - return MemoryOps.readBytesAsInt64(this, len); - } + int readerIdx = readerIndex; // use subtract to avoid overflow int remaining = size - readerIdx; @@ -3534,9 +3319,7 @@ public void read(ByteBuffer dst, int len) { * is optimized for small size, it's faster than {@link #readVarUInt32}. */ public int readBinarySize() { - if (AndroidSupport.IS_ANDROID) { - return MemoryOps.readBinarySize(this); - } + int binarySize; int readIdx = readerIndex; if (size - readIdx >= 5) { @@ -3613,9 +3396,7 @@ public byte[] readBytesAndSize() { * prefix. */ public void readByteArrayPayload(byte[] values, int numBytes) { - if (AndroidSupport.IS_ANDROID) { - MemoryOps.readByteArrayPayload(this, values, numBytes); - } else { + int readerIdx = readerIndex; if (readerIdx > size - numBytes) { streamReader.readTo(values, 0, numBytes); @@ -3623,7 +3404,7 @@ public void readByteArrayPayload(byte[] values, int numBytes) { } readBytesToArray(address + readerIdx, values, BYTE_ARRAY_OFFSET, numBytes); readerIndex = readerIdx + numBytes; - } + } /** @@ -3632,9 +3413,7 @@ public void readByteArrayPayload(byte[] values, int numBytes) { * prefix. */ public void readBooleanArrayPayload(boolean[] values, int numBytes) { - if (AndroidSupport.IS_ANDROID) { - MemoryOps.readBooleanArrayPayload(this, values, numBytes); - } else { + int readerIdx = readerIndex; if (readerIdx > size - numBytes) { streamReader.readBooleans(values, 0, numBytes); @@ -3642,7 +3421,7 @@ public void readBooleanArrayPayload(boolean[] values, int numBytes) { } readBooleansToArray(address + readerIdx, values, BOOLEAN_ARRAY_OFFSET, numBytes); readerIndex = readerIdx + numBytes; - } + } /** @@ -3651,9 +3430,7 @@ public void readBooleanArrayPayload(boolean[] values, int numBytes) { * prefix. */ public void readCharArrayPayload(char[] values, int numBytes) { - if (AndroidSupport.IS_ANDROID) { - MemoryOps.readCharArrayPayload(this, values, numBytes); - } else { + int readerIdx = readerIndex; if (readerIdx > size - numBytes) { streamReader.readChars(values, 0, numBytes >>> 1); @@ -3661,7 +3438,7 @@ public void readCharArrayPayload(char[] values, int numBytes) { } readCharsToArray(address + readerIdx, values, CHAR_ARRAY_OFFSET, numBytes); readerIndex = readerIdx + numBytes; - } + } /** @@ -3670,9 +3447,7 @@ public void readCharArrayPayload(char[] values, int numBytes) { * prefix. */ public void readInt16ArrayPayload(short[] values, int numBytes) { - if (AndroidSupport.IS_ANDROID) { - MemoryOps.readInt16ArrayPayload(this, values, numBytes); - } else { + int readerIdx = readerIndex; if (readerIdx > size - numBytes) { streamReader.readShorts(values, 0, numBytes >>> 1); @@ -3680,7 +3455,7 @@ public void readInt16ArrayPayload(short[] values, int numBytes) { } readShortsToArray(address + readerIdx, values, SHORT_ARRAY_OFFSET, numBytes); readerIndex = readerIdx + numBytes; - } + } /** @@ -3689,9 +3464,7 @@ public void readInt16ArrayPayload(short[] values, int numBytes) { * prefix. */ public void readInt32ArrayPayload(int[] values, int numBytes) { - if (AndroidSupport.IS_ANDROID) { - MemoryOps.readInt32ArrayPayload(this, values, numBytes); - } else { + int readerIdx = readerIndex; if (readerIdx > size - numBytes) { streamReader.readInts(values, 0, numBytes >>> 2); @@ -3699,7 +3472,7 @@ public void readInt32ArrayPayload(int[] values, int numBytes) { } readIntsToArray(address + readerIdx, values, INT_ARRAY_OFFSET, numBytes); readerIndex = readerIdx + numBytes; - } + } /** @@ -3708,9 +3481,7 @@ public void readInt32ArrayPayload(int[] values, int numBytes) { * prefix. */ public void readInt64ArrayPayload(long[] values, int numBytes) { - if (AndroidSupport.IS_ANDROID) { - MemoryOps.readInt64ArrayPayload(this, values, numBytes); - } else { + int readerIdx = readerIndex; if (readerIdx > size - numBytes) { streamReader.readLongs(values, 0, numBytes >>> 3); @@ -3718,7 +3489,7 @@ public void readInt64ArrayPayload(long[] values, int numBytes) { } readLongsToArray(address + readerIdx, values, LONG_ARRAY_OFFSET, numBytes); readerIndex = readerIdx + numBytes; - } + } /** @@ -3727,9 +3498,7 @@ public void readInt64ArrayPayload(long[] values, int numBytes) { * prefix. */ public void readFloat32ArrayPayload(float[] values, int numBytes) { - if (AndroidSupport.IS_ANDROID) { - MemoryOps.readFloat32ArrayPayload(this, values, numBytes); - } else { + int readerIdx = readerIndex; if (readerIdx > size - numBytes) { streamReader.readFloats(values, 0, numBytes >>> 2); @@ -3737,7 +3506,7 @@ public void readFloat32ArrayPayload(float[] values, int numBytes) { } readFloatsToArray(address + readerIdx, values, FLOAT_ARRAY_OFFSET, numBytes); readerIndex = readerIdx + numBytes; - } + } /** @@ -3746,9 +3515,7 @@ public void readFloat32ArrayPayload(float[] values, int numBytes) { * prefix. */ public void readFloat64ArrayPayload(double[] values, int numBytes) { - if (AndroidSupport.IS_ANDROID) { - MemoryOps.readFloat64ArrayPayload(this, values, numBytes); - } else { + int readerIdx = readerIndex; if (readerIdx > size - numBytes) { streamReader.readDoubles(values, 0, numBytes >>> 3); @@ -3756,13 +3523,11 @@ public void readFloat64ArrayPayload(double[] values, int numBytes) { } readDoublesToArray(address + readerIdx, values, DOUBLE_ARRAY_OFFSET, numBytes); readerIndex = readerIdx + numBytes; - } + } public void readBooleans(boolean[] values, int offset, int numElements) { - if (AndroidSupport.IS_ANDROID) { - MemoryOps.readBooleans(this, values, offset, numElements); - } else { + if ((offset | numElements | (offset + numElements) | (values.length - offset - numElements)) < 0) { throwOOBException(); @@ -3774,7 +3539,7 @@ public void readBooleans(boolean[] values, int offset, int numElements) { int readerIdx = readerIndex; readBooleansToArray(address + readerIdx, values, BOOLEAN_ARRAY_OFFSET + offset, numElements); readerIndex = readerIdx + numElements; - } + } public void readChars(char[] chars, int numElements) { @@ -3782,9 +3547,7 @@ public void readChars(char[] chars, int numElements) { } public void readChars(char[] chars, int offset, int numElements) { - if (AndroidSupport.IS_ANDROID) { - MemoryOps.readChars(this, chars, offset, numElements); - } else { + if ((offset | numElements | (offset + numElements) | (chars.length - offset - numElements)) < 0) { throwOOBException(); @@ -3797,7 +3560,7 @@ public void readChars(char[] chars, int offset, int numElements) { int readerIdx = readerIndex; readCharsToArray(address + readerIdx, chars, CHAR_ARRAY_OFFSET + offset, numBytes); readerIndex = readerIdx + numBytes; - } + } @CodegenInvoke @@ -3810,9 +3573,7 @@ public char[] readCharsAndSize() { } public void readShorts(short[] values, int offset, int numElements) { - if (AndroidSupport.IS_ANDROID) { - MemoryOps.readShorts(this, values, offset, numElements); - } else { + if ((offset | numElements | (offset + numElements) | (values.length - offset - numElements)) < 0) { throwOOBException(); @@ -3825,13 +3586,11 @@ public void readShorts(short[] values, int offset, int numElements) { int readerIdx = readerIndex; readShortsToArray(address + readerIdx, values, SHORT_ARRAY_OFFSET + offset, numBytes); readerIndex = readerIdx + numBytes; - } + } public void readInts(int[] values, int offset, int numElements) { - if (AndroidSupport.IS_ANDROID) { - MemoryOps.readInts(this, values, offset, numElements); - } else { + if ((offset | numElements | (offset + numElements) | (values.length - offset - numElements)) < 0) { throwOOBException(); @@ -3844,13 +3603,11 @@ public void readInts(int[] values, int offset, int numElements) { int readerIdx = readerIndex; readIntsToArray(address + readerIdx, values, INT_ARRAY_OFFSET + offset, numBytes); readerIndex = readerIdx + numBytes; - } + } public void readLongs(long[] values, int offset, int numElements) { - if (AndroidSupport.IS_ANDROID) { - MemoryOps.readLongs(this, values, offset, numElements); - } else { + if ((offset | numElements | (offset + numElements) | (values.length - offset - numElements)) < 0) { throwOOBException(); @@ -3863,13 +3620,11 @@ public void readLongs(long[] values, int offset, int numElements) { int readerIdx = readerIndex; readLongsToArray(address + readerIdx, values, LONG_ARRAY_OFFSET + offset, numBytes); readerIndex = readerIdx + numBytes; - } + } public void readFloats(float[] values, int offset, int numElements) { - if (AndroidSupport.IS_ANDROID) { - MemoryOps.readFloats(this, values, offset, numElements); - } else { + if ((offset | numElements | (offset + numElements) | (values.length - offset - numElements)) < 0) { throwOOBException(); @@ -3882,13 +3637,11 @@ public void readFloats(float[] values, int offset, int numElements) { int readerIdx = readerIndex; readFloatsToArray(address + readerIdx, values, FLOAT_ARRAY_OFFSET + offset, numBytes); readerIndex = readerIdx + numBytes; - } + } public void readDoubles(double[] values, int offset, int numElements) { - if (AndroidSupport.IS_ANDROID) { - MemoryOps.readDoubles(this, values, offset, numElements); - } else { + if ((offset | numElements | (offset + numElements) | (values.length - offset - numElements)) < 0) { throwOOBException(); @@ -3901,7 +3654,7 @@ public void readDoubles(double[] values, int offset, int numElements) { int readerIdx = readerIndex; readDoublesToArray(address + readerIdx, values, DOUBLE_ARRAY_OFFSET + offset, numBytes); readerIndex = readerIdx + numBytes; - } + } public void checkReadableBytes(int minimumReadableBytes) { @@ -3928,9 +3681,7 @@ public byte[] getRemainingBytes() { // ------------------------- Read Methods Finished ------------------------------------- public void copyTo(int offset, MemoryBuffer target, int targetOffset, int numBytes) { - if (AndroidSupport.IS_ANDROID) { - MemoryOps.copyTo(this, offset, target, targetOffset, numBytes); - } else { + final byte[] thisHeapRef = this.heapMemory; final byte[] otherHeapRef = target.heapMemory; final long thisPointer = this.address + offset; @@ -3953,7 +3704,7 @@ public void copyTo(int offset, MemoryBuffer target, int targetOffset, int numByt "offset=%d, targetOffset=%d, numBytes=%d, address=%d, targetAddress=%d", offset, targetOffset, numBytes, this.address, target.address)); } - } + } private boolean sameBufferOverlap( @@ -3979,75 +3730,59 @@ public void copyFrom(int offset, MemoryBuffer source, int sourcePointer, int num } public void copyToByteArray(int offset, byte[] target, int targetOffset, int numBytes) { - if (AndroidSupport.IS_ANDROID) { - MemoryOps.copyToByteArray(this, offset, target, targetOffset, numBytes); - } else { + checkArrayCopy(offset, targetOffset, target.length, numBytes, 0); readBytesToArray(address + offset, target, BYTE_ARRAY_OFFSET + targetOffset, numBytes); - } + } public void copyToBooleanArray(int offset, boolean[] target, int targetOffset, int numBytes) { - if (AndroidSupport.IS_ANDROID) { - MemoryOps.copyToBooleanArray(this, offset, target, targetOffset, numBytes); - } else { + checkArrayCopy(offset, targetOffset, target.length, numBytes, 0); readBooleansToArray(address + offset, target, BOOLEAN_ARRAY_OFFSET + targetOffset, numBytes); - } + } public void copyToCharArray(int offset, char[] target, int targetOffset, int numBytes) { - if (AndroidSupport.IS_ANDROID) { - MemoryOps.copyToCharArray(this, offset, target, targetOffset, numBytes); - } else { + checkArrayCopy(offset, targetOffset, target.length, numBytes, 1); readCharsToArray(address + offset, target, CHAR_ARRAY_OFFSET + targetOffset, numBytes); - } + } public void copyToShortArray(int offset, short[] target, int targetOffset, int numBytes) { - if (AndroidSupport.IS_ANDROID) { - MemoryOps.copyToShortArray(this, offset, target, targetOffset, numBytes); - } else { + checkArrayCopy(offset, targetOffset, target.length, numBytes, 1); readShortsToArray(address + offset, target, SHORT_ARRAY_OFFSET + targetOffset, numBytes); - } + } public void copyToIntArray(int offset, int[] target, int targetOffset, int numBytes) { - if (AndroidSupport.IS_ANDROID) { - MemoryOps.copyToIntArray(this, offset, target, targetOffset, numBytes); - } else { + checkArrayCopy(offset, targetOffset, target.length, numBytes, 2); readIntsToArray(address + offset, target, INT_ARRAY_OFFSET + targetOffset, numBytes); - } + } public void copyToLongArray(int offset, long[] target, int targetOffset, int numBytes) { - if (AndroidSupport.IS_ANDROID) { - MemoryOps.copyToLongArray(this, offset, target, targetOffset, numBytes); - } else { + checkArrayCopy(offset, targetOffset, target.length, numBytes, 3); readLongsToArray(address + offset, target, LONG_ARRAY_OFFSET + targetOffset, numBytes); - } + } public void copyToFloatArray(int offset, float[] target, int targetOffset, int numBytes) { - if (AndroidSupport.IS_ANDROID) { - MemoryOps.copyToFloatArray(this, offset, target, targetOffset, numBytes); - } else { + checkArrayCopy(offset, targetOffset, target.length, numBytes, 2); readFloatsToArray(address + offset, target, FLOAT_ARRAY_OFFSET + targetOffset, numBytes); - } + } public void copyToDoubleArray(int offset, double[] target, int targetOffset, int numBytes) { - if (AndroidSupport.IS_ANDROID) { - MemoryOps.copyToDoubleArray(this, offset, target, targetOffset, numBytes); - } else { + checkArrayCopy(offset, targetOffset, target.length, numBytes, 3); readDoublesToArray(address + offset, target, DOUBLE_ARRAY_OFFSET + targetOffset, numBytes); - } + } private void checkArrayCopy( @@ -4065,76 +3800,60 @@ private void checkArrayCopy( } public void copyFromByteArray(int offset, byte[] source, int sourceOffset, int numBytes) { - if (AndroidSupport.IS_ANDROID) { - MemoryOps.copyFromByteArray(this, offset, source, sourceOffset, numBytes); - } else { + checkArrayCopy(offset, sourceOffset, source.length, numBytes, 0); writeBytesFromArray(address + offset, source, BYTE_ARRAY_OFFSET + sourceOffset, numBytes); - } + } public void copyFromBooleanArray(int offset, boolean[] source, int sourceOffset, int numBytes) { - if (AndroidSupport.IS_ANDROID) { - MemoryOps.copyFromBooleanArray(this, offset, source, sourceOffset, numBytes); - } else { + checkArrayCopy(offset, sourceOffset, source.length, numBytes, 0); writeBooleansFromArray( address + offset, source, BOOLEAN_ARRAY_OFFSET + sourceOffset, numBytes); - } + } public void copyFromCharArray(int offset, char[] source, int sourceOffset, int numBytes) { - if (AndroidSupport.IS_ANDROID) { - MemoryOps.copyFromCharArray(this, offset, source, sourceOffset, numBytes); - } else { + checkArrayCopy(offset, sourceOffset, source.length, numBytes, 1); writeCharsFromArray(address + offset, source, CHAR_ARRAY_OFFSET + sourceOffset, numBytes); - } + } public void copyFromShortArray(int offset, short[] source, int sourceOffset, int numBytes) { - if (AndroidSupport.IS_ANDROID) { - MemoryOps.copyFromShortArray(this, offset, source, sourceOffset, numBytes); - } else { + checkArrayCopy(offset, sourceOffset, source.length, numBytes, 1); writeShortsFromArray(address + offset, source, SHORT_ARRAY_OFFSET + sourceOffset, numBytes); - } + } public void copyFromIntArray(int offset, int[] source, int sourceOffset, int numBytes) { - if (AndroidSupport.IS_ANDROID) { - MemoryOps.copyFromIntArray(this, offset, source, sourceOffset, numBytes); - } else { + checkArrayCopy(offset, sourceOffset, source.length, numBytes, 2); writeIntsFromArray(address + offset, source, INT_ARRAY_OFFSET + sourceOffset, numBytes); - } + } public void copyFromLongArray(int offset, long[] source, int sourceOffset, int numBytes) { - if (AndroidSupport.IS_ANDROID) { - MemoryOps.copyFromLongArray(this, offset, source, sourceOffset, numBytes); - } else { + checkArrayCopy(offset, sourceOffset, source.length, numBytes, 3); writeLongsFromArray(address + offset, source, LONG_ARRAY_OFFSET + sourceOffset, numBytes); - } + } public void copyFromFloatArray(int offset, float[] source, int sourceOffset, int numBytes) { - if (AndroidSupport.IS_ANDROID) { - MemoryOps.copyFromFloatArray(this, offset, source, sourceOffset, numBytes); - } else { + checkArrayCopy(offset, sourceOffset, source.length, numBytes, 2); writeFloatsFromArray(address + offset, source, FLOAT_ARRAY_OFFSET + sourceOffset, numBytes); - } + } public void copyFromDoubleArray(int offset, double[] source, int sourceOffset, int numBytes) { - if (AndroidSupport.IS_ANDROID) { - MemoryOps.copyFromDoubleArray(this, offset, source, sourceOffset, numBytes); - } else { + checkArrayCopy(offset, sourceOffset, source.length, numBytes, 3); writeDoublesFromArray(address + offset, source, DOUBLE_ARRAY_OFFSET + sourceOffset, numBytes); - } + } public byte[] getBytes(int index, int length) { @@ -4222,9 +3941,7 @@ public boolean equalTo(MemoryBuffer buf2, int offset1, int offset2, int len) { if (len == 0) { return buf2 != null; } - if (AndroidSupport.IS_ANDROID) { - return MemoryOps.equalTo(this, buf2, offset1, offset2, len); - } + final long pos1 = address + offset1; final long pos2 = buf2.address + offset2; checkArgument(pos1 < addressLimit); @@ -4249,9 +3966,7 @@ public boolean equalTo(byte[] bytes, int bytesOffset, int offset, int len) { if (len == 0) { return true; } - if (AndroidSupport.IS_ANDROID) { - return MemoryOps.equalTo(this, bytes, bytesOffset, offset, len); - } + final long pos = address + offset; return unsafeEqualTo(this, heapMemory, pos, this, bytes, BYTE_ARRAY_OFFSET + bytesOffset, len); } @@ -4405,9 +4120,7 @@ public static MemoryBuffer fromByteArray(byte[] buffer) { * @param buffer a direct buffer or heap buffer */ public static MemoryBuffer fromByteBuffer(ByteBuffer buffer) { - if (AndroidSupport.IS_ANDROID) { - return MemoryOps.fromByteBuffer(buffer); - } else if (buffer.isDirect()) { + if (buffer.isDirect()) { return new MemoryBuffer(buffer.position(), buffer.remaining(), buffer); } else if (buffer.hasArray()) { int offset = buffer.arrayOffset() + buffer.position(); @@ -4422,9 +4135,7 @@ public static MemoryBuffer fromByteBuffer(ByteBuffer buffer) { public static MemoryBuffer fromDirectByteBuffer( ByteBuffer buffer, int size, ForyStreamReader streamReader) { - if (AndroidSupport.IS_ANDROID) { - return MemoryOps.directByteBufferUnsupported(); - } + long offHeapAddress = buffer.position(); return new MemoryBuffer(offHeapAddress, size, buffer, streamReader); } diff --git a/java/fory-core/src/main/java25/org/apache/fory/platform/internal/DefineClass.java b/java/fory-core/src/main/java25/org/apache/fory/platform/internal/DefineClass.java index 4dea13dfcf..5afebee02f 100644 --- a/java/fory-core/src/main/java25/org/apache/fory/platform/internal/DefineClass.java +++ b/java/fory-core/src/main/java25/org/apache/fory/platform/internal/DefineClass.java @@ -25,7 +25,6 @@ import java.lang.invoke.MethodType; import java.security.ProtectionDomain; import org.apache.fory.annotation.Internal; -import org.apache.fory.platform.AndroidSupport; import org.apache.fory.platform.JdkVersion; import org.apache.fory.util.Preconditions; @@ -40,10 +39,6 @@ public static Class defineClass( ClassLoader loader, ProtectionDomain domain, byte[] bytecodes) { - if (AndroidSupport.IS_ANDROID) { - throw new UnsupportedOperationException( - "Runtime bytecode loading is unsupported on Android."); - } Preconditions.checkNotNull(loader); Preconditions.checkArgument(JdkVersion.MAJOR_VERSION >= 8); if (neighbor != null && JdkVersion.MAJOR_VERSION >= 9) { @@ -81,10 +76,6 @@ public static Class defineClass( } public static Class defineHiddenNestmate(Class neighbor, byte[] bytecodes) { - if (AndroidSupport.IS_ANDROID) { - throw new UnsupportedOperationException( - "Runtime bytecode loading is unsupported on Android."); - } Preconditions.checkNotNull(neighbor); Preconditions.checkNotNull(bytecodes); try { diff --git a/java/fory-core/src/main/java25/org/apache/fory/platform/internal/_JDKAccess.java b/java/fory-core/src/main/java25/org/apache/fory/platform/internal/_JDKAccess.java deleted file mode 100644 index e4153744cc..0000000000 --- a/java/fory-core/src/main/java25/org/apache/fory/platform/internal/_JDKAccess.java +++ /dev/null @@ -1,809 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.fory.platform.internal; - -import java.io.ObjectInputStream; -import java.io.ObjectOutputStream; -import java.lang.invoke.CallSite; -import java.lang.invoke.LambdaConversionException; -import java.lang.invoke.LambdaMetafactory; -import java.lang.invoke.MethodHandle; -import java.lang.invoke.MethodHandles; -import java.lang.invoke.MethodHandles.Lookup; -import java.lang.invoke.MethodType; -import java.lang.invoke.SerializedLambda; -import java.lang.invoke.VarHandle; -import java.lang.reflect.Field; -import java.lang.reflect.Method; -import java.lang.reflect.Modifier; -import java.lang.reflect.Proxy; -import java.nio.charset.StandardCharsets; -import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.ArrayBlockingQueue; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.function.BiConsumer; -import java.util.function.BiFunction; -import java.util.function.Consumer; -import java.util.function.Function; -import java.util.function.Predicate; -import java.util.function.ToDoubleFunction; -import java.util.function.ToIntFunction; -import java.util.function.ToLongFunction; -import org.apache.fory.collection.ClassValueCache; -import org.apache.fory.collection.Tuple2; -import org.apache.fory.platform.GraalvmSupport; -import org.apache.fory.platform.JdkVersion; -import org.apache.fory.type.TypeUtils; -import org.apache.fory.util.ExceptionUtils; -import org.apache.fory.util.Preconditions; -import org.apache.fory.util.function.ToByteFunction; -import org.apache.fory.util.function.ToCharFunction; -import org.apache.fory.util.function.ToFloatFunction; -import org.apache.fory.util.function.ToShortFunction; - -/** JDK internals access for the JDK25 multi-release runtime. */ -// CHECKSTYLE.OFF:TypeName -public class _JDKAccess { - // CHECKSTYLE.ON:TypeName - public static final boolean IS_OPEN_J9; - public static final boolean JDK_INTERNAL_FIELD_ACCESS; - public static final boolean JDK_LANG_FIELD_ACCESS; - public static final boolean JDK_STRING_FIELD_ACCESS; - public static final boolean JDK_COLLECTION_FIELD_ACCESS; - public static final boolean JDK_CONCURRENT_FIELD_ACCESS; - public static final boolean JDK_PROXY_FIELD_ACCESS; - - private static final ClassValueCache lookupCache = ClassValueCache.newClassKeyCache(32); - - public static final boolean STRING_VALUE_FIELD_IS_CHARS; - public static final boolean STRING_VALUE_FIELD_IS_BYTES; - public static final boolean STRING_HAS_COUNT_OFFSET; - public static final long STRING_VALUE_FIELD_OFFSET = -1; - public static final long STRING_COUNT_FIELD_OFFSET = -1; - public static final long STRING_OFFSET_FIELD_OFFSET = -1; - public static final long STRING_CODER_FIELD_OFFSET = -1; - private static final VarHandle STRING_VALUE_HANDLE; - private static final VarHandle STRING_CODER_HANDLE; - private static final VarHandle STRING_COUNT_HANDLE; - private static final VarHandle STRING_OFFSET_HANDLE; - - static { - String jmvName = System.getProperty("java.vm.name", ""); - IS_OPEN_J9 = jmvName.contains("OpenJ9"); - } - - static { - try { - Field valueField = String.class.getDeclaredField("value"); - STRING_VALUE_FIELD_IS_CHARS = valueField.getType() == char[].class; - STRING_VALUE_FIELD_IS_BYTES = valueField.getType() == byte[].class; - Field countField = getStringFieldNullable("count"); - Field offsetField = getStringFieldNullable("offset"); - if (countField != null || offsetField != null) { - Preconditions.checkArgument( - countField != null && offsetField != null, "Current jdk not supported"); - Preconditions.checkArgument( - countField.getType() == int.class && offsetField.getType() == int.class, - "Current jdk not supported"); - STRING_HAS_COUNT_OFFSET = true; - } else { - STRING_HAS_COUNT_OFFSET = false; - } - - StringHandles stringHandles = initStringHandles(valueField.getType(), countField, offsetField); - - JDK_LANG_FIELD_ACCESS = canAccess(String.class); - JDK_STRING_FIELD_ACCESS = stringHandles != null; - JDK_COLLECTION_FIELD_ACCESS = canAccess("java.util.Collections$SynchronizedCollection"); - JDK_CONCURRENT_FIELD_ACCESS = - canAccess(ArrayBlockingQueue.class) && canAccess(LinkedBlockingQueue.class); - JDK_PROXY_FIELD_ACCESS = canAccess(Proxy.class); - JDK_INTERNAL_FIELD_ACCESS = JDK_STRING_FIELD_ACCESS; - - STRING_VALUE_HANDLE = stringHandles == null ? null : stringHandles.value; - STRING_CODER_HANDLE = stringHandles == null ? null : stringHandles.coder; - STRING_COUNT_HANDLE = stringHandles == null ? null : stringHandles.count; - STRING_OFFSET_HANDLE = stringHandles == null ? null : stringHandles.offset; - } catch (NoSuchFieldException e) { - throw new RuntimeException(e); - } - } - - private static Field getStringFieldNullable(String fieldName) { - try { - return String.class.getDeclaredField(fieldName); - } catch (NoSuchFieldException e) { - return null; - } - } - - private static StringHandles initStringHandles( - Class stringValueType, Field countField, Field offsetField) { - try { - Lookup stringLookup = _Lookup._trustedLookup(String.class); - return new StringHandles( - stringLookup.findVarHandle(String.class, "value", stringValueType), - STRING_VALUE_FIELD_IS_BYTES - ? stringLookup.findVarHandle(String.class, "coder", byte.class) - : null, - countField == null ? null : stringLookup.findVarHandle(String.class, "count", int.class), - offsetField == null - ? null - : stringLookup.findVarHandle(String.class, "offset", int.class)); - } catch (Throwable ignored) { - return null; - } - } - - private static boolean canAccess(String className) { - try { - return canAccess(Class.forName(className)); - } catch (Throwable ignored) { - return false; - } - } - - private static boolean canAccess(Class type) { - try { - _Lookup._trustedLookup(type); - return true; - } catch (Throwable ignored) { - return false; - } - } - - private static class StringHandles { - private final VarHandle value; - private final VarHandle coder; - private final VarHandle count; - private final VarHandle offset; - - private StringHandles(VarHandle value, VarHandle coder, VarHandle count, VarHandle offset) { - this.value = value; - this.coder = coder; - this.count = count; - this.offset = offset; - } - } - - // The root native-image configuration names this root lazy helper. Keep a same-named JDK25 - // shadow so multi-release class lookup does not fall back to the root Unsafe offset helper. - private static class StringCoderField {} - - public static Object getStringValue(String value) { - checkStringAccess("String.value"); - return STRING_VALUE_HANDLE.get(value); - } - - public static byte getStringCoder(String value) { - checkStringAccess("String.coder"); - if (STRING_CODER_HANDLE == null) { - throw new UnsupportedOperationException("String.coder is not available on this JDK"); - } - return (byte) STRING_CODER_HANDLE.get(value); - } - - public static int getStringOffset(String value) { - checkStringAccess("String.offset"); - if (STRING_OFFSET_HANDLE == null) { - throw new UnsupportedOperationException("String.offset is not available on this JDK"); - } - return (int) STRING_OFFSET_HANDLE.get(value); - } - - public static int getStringCount(String value) { - checkStringAccess("String.count"); - if (STRING_COUNT_HANDLE == null) { - throw new UnsupportedOperationException("String.count is not available on this JDK"); - } - return (int) STRING_COUNT_HANDLE.get(value); - } - - // CHECKSTYLE.OFF:MethodName - - public static Lookup _trustedLookup(Class objectClass) { - // CHECKSTYLE.ON:MethodName - if (GraalvmSupport.isGraalBuildTime()) { - return _Lookup._trustedLookup(objectClass); - } - return lookupCache.get(objectClass, () -> _Lookup._trustedLookup(objectClass)); - } - - public static MethodHandle readResolveHandle(Class cls, Method method) - throws IllegalAccessException { - try { - return _trustedLookup(cls).unreflect(method); - } catch (IllegalArgumentException e) { - if (cls != SerializedLambda.class) { - throw e; - } - // JDK25 rejects SerializedLambda itself as a privateLookupIn target. Reflective access still - // honors the same java.base/java.lang.invoke open requirement and avoids serializer-level - // versioning. - try { - method.setAccessible(true); - } catch (RuntimeException inaccessible) { - throw new IllegalStateException( - "SerializedLambda readResolve requires java.base/java.lang.invoke to be open to " - + "org.apache.fory.core", - inaccessible); - } - return MethodHandles.lookup().unreflect(method); - } - } - - private static final byte LATIN1 = 0; - private static final Byte LATIN1_BOXED = LATIN1; - private static final byte UTF16 = 1; - private static final Byte UTF16_BOXED = UTF16; - private static final Lookup STRING_LOOK_UP = - JDK_STRING_FIELD_ACCESS ? _trustedLookup(String.class) : null; - private static final MethodHandle STRING_ZERO_COPY_CTR_HANDLE = - JDK_STRING_FIELD_ACCESS ? getJavaStringZeroCopyCtrHandle() : null; - private static final BiFunction CHARS_STRING_ZERO_COPY_CTR = - JDK_STRING_FIELD_ACCESS ? getCharsStringZeroCopyCtr() : null; - private static final BiFunction BYTES_STRING_ZERO_COPY_CTR = - JDK_STRING_FIELD_ACCESS ? getBytesStringZeroCopyCtr() : null; - private static final Function LATIN_BYTES_STRING_ZERO_COPY_CTR = - JDK_STRING_FIELD_ACCESS ? getLatinBytesStringZeroCopyCtr() : null; - - public static String newCharsStringZeroCopy(char[] data) { - if (!JDK_STRING_FIELD_ACCESS) { - return new String(data); - } - if (!STRING_VALUE_FIELD_IS_CHARS) { - throw new IllegalStateException("String value isn't char[], current java isn't supported"); - } - if (CHARS_STRING_ZERO_COPY_CTR == null) { - return newCharsStringByHandle(data); - } - return CHARS_STRING_ZERO_COPY_CTR.apply(data, Boolean.TRUE); - } - - private static String newCharsStringByHandle(char[] data) { - MethodHandle handle = STRING_ZERO_COPY_CTR_HANDLE; - if (handle == null) { - return new String(data); - } - try { - return (String) handle.invokeExact(data, true); - } catch (Throwable ignored) { - return new String(data); - } - } - - public static String newBytesStringZeroCopy(byte coder, byte[] data) { - if (!JDK_STRING_FIELD_ACCESS) { - return newBytesStringSlow(coder, data); - } - if (coder == LATIN1) { - if (LATIN_BYTES_STRING_ZERO_COPY_CTR != null) { - return LATIN_BYTES_STRING_ZERO_COPY_CTR.apply(data); - } else if (BYTES_STRING_ZERO_COPY_CTR == null) { - return newBytesStringByHandle(coder, data); - } - return BYTES_STRING_ZERO_COPY_CTR.apply(data, LATIN1_BOXED); - } else if (coder == UTF16) { - if (BYTES_STRING_ZERO_COPY_CTR == null) { - return newBytesStringByHandle(coder, data); - } - return BYTES_STRING_ZERO_COPY_CTR.apply(data, UTF16_BOXED); - } else { - if (BYTES_STRING_ZERO_COPY_CTR == null) { - return newBytesStringSlow(coder, data); - } - return BYTES_STRING_ZERO_COPY_CTR.apply(data, coder); - } - } - - private static String newBytesStringSlow(byte coder, byte[] data) { - if (coder == LATIN1) { - return new String(data, StandardCharsets.ISO_8859_1); - } else if (coder == UTF16) { - char[] chars = new char[data.length >> 1]; - for (int i = 0, j = 0; i < data.length; i += 2) { - chars[j++] = (char) ((data[i] & 0xff) | ((data[i + 1] & 0xff) << 8)); - } - return new String(chars); - } else { - return new String(data, StandardCharsets.UTF_8); - } - } - - private static String newBytesStringByHandle(byte coder, byte[] data) { - MethodHandle handle = STRING_ZERO_COPY_CTR_HANDLE; - if (handle == null) { - return newBytesStringSlow(coder, data); - } - try { - return (String) handle.invokeExact(data, coder); - } catch (Throwable ignored) { - return newBytesStringSlow(coder, data); - } - } - - private static BiFunction getCharsStringZeroCopyCtr() { - if (!STRING_VALUE_FIELD_IS_CHARS) { - return null; - } - MethodHandle handle = STRING_ZERO_COPY_CTR_HANDLE; - if (handle == null) { - return null; - } - try { - CallSite callSite = - LambdaMetafactory.metafactory( - STRING_LOOK_UP, - "apply", - MethodType.methodType(BiFunction.class), - handle.type().generic(), - handle, - handle.type()); - return (BiFunction) callSite.getTarget().invokeExact(); - } catch (Throwable e) { - return null; - } - } - - private static BiFunction getBytesStringZeroCopyCtr() { - if (!STRING_VALUE_FIELD_IS_BYTES) { - return null; - } - MethodHandle handle = STRING_ZERO_COPY_CTR_HANDLE; - if (handle == null) { - return null; - } - try { - MethodType instantiatedMethodType = - MethodType.methodType(handle.type().returnType(), new Class[] {byte[].class, Byte.class}); - CallSite callSite = - LambdaMetafactory.metafactory( - STRING_LOOK_UP, - "apply", - MethodType.methodType(BiFunction.class), - handle.type().generic(), - handle, - instantiatedMethodType); - return (BiFunction) callSite.getTarget().invokeExact(); - } catch (Throwable e) { - return null; - } - } - - private static Function getLatinBytesStringZeroCopyCtr() { - if (!STRING_VALUE_FIELD_IS_BYTES || STRING_LOOK_UP == null) { - return null; - } - try { - Class clazz = Class.forName("java.lang.StringCoding"); - Lookup caller = STRING_LOOK_UP.in(clazz); - MethodHandle handle = - caller.findStatic( - clazz, "newStringLatin1", MethodType.methodType(String.class, byte[].class)); - return makeFunction(caller, handle, Function.class); - } catch (Throwable e) { - return null; - } - } - - private static MethodHandle getJavaStringZeroCopyCtrHandle() { - Preconditions.checkArgument(JdkVersion.MAJOR_VERSION >= 8); - if (STRING_LOOK_UP == null) { - return null; - } - try { - if (STRING_VALUE_FIELD_IS_CHARS) { - return STRING_LOOK_UP.findConstructor( - String.class, MethodType.methodType(void.class, char[].class, boolean.class)); - } else { - return STRING_LOOK_UP.findConstructor( - String.class, MethodType.methodType(void.class, byte[].class, byte.class)); - } - } catch (Exception e) { - return null; - } - } - - private static void checkStringAccess(String target) { - if (!JDK_STRING_FIELD_ACCESS) { - throw new UnsupportedOperationException( - target - + " private access is unavailable; open java.base/java.lang.invoke to " - + "org.apache.fory.core"); - } - } - - private static final ClassValueCache serializationMethodsCache = - ClassValueCache.newClassKeyCache(32); - - private static final class SerializationMethods { - private final Method writeObject; - private final Method readObject; - private final Method readObjectNoData; - private final Method writeReplace; - private final Method readResolve; - - private SerializationMethods(Class type) { - writeObject = - getPrivateMethod(type, "writeObject", void.class, ObjectOutputStream.class); - readObject = getPrivateMethod(type, "readObject", void.class, ObjectInputStream.class); - readObjectNoData = getPrivateMethod(type, "readObjectNoData", void.class); - writeReplace = getInheritableObjectMethod(type, "writeReplace"); - readResolve = getInheritableObjectMethod(type, "readResolve"); - } - } - - private static SerializationMethods serializationMethods(Class type) { - return serializationMethodsCache.get(type, () -> new SerializationMethods(type)); - } - - private static Method getPrivateMethod( - Class type, String methodName, Class returnType, Class... parameterTypes) { - try { - Method method = type.getDeclaredMethod(methodName, parameterTypes); - int modifiers = method.getModifiers(); - if (Modifier.isPrivate(modifiers) - && !Modifier.isStatic(modifiers) - && method.getReturnType() == returnType) { - return method; - } - } catch (NoSuchMethodException | SecurityException e) { - ExceptionUtils.ignore(e); - } - return null; - } - - private static Method getInheritableObjectMethod(Class type, String methodName) { - Class cls = type; - while (cls != null) { - try { - Method method = cls.getDeclaredMethod(methodName); - int modifiers = method.getModifiers(); - if (!Modifier.isStatic(modifiers) && method.getReturnType() == Object.class) { - return method; - } - return null; - } catch (NoSuchMethodException e) { - ExceptionUtils.ignore(e); - } catch (SecurityException e) { - return null; - } - cls = cls.getSuperclass(); - } - return null; - } - - public static Method getSerializationWriteObjectMethod(Class type) { - return serializationMethods(type).writeObject; - } - - public static Method getSerializationReadObjectMethod(Class type) { - return serializationMethods(type).readObject; - } - - public static Method getSerializationReadObjectNoDataMethod(Class type) { - return serializationMethods(type).readObjectNoData; - } - - public static MethodHandle getSerializationDefaultReadObjectHandle(Class type) { - return null; - } - - public static Method getSerializationWriteReplaceMethod(Class type) { - return serializationMethods(type).writeReplace; - } - - public static Method getSerializationReadResolveMethod(Class type) { - return serializationMethods(type).readResolve; - } - - public static boolean isSerializationHookLookupAvailable() { - return true; - } - - public static T tryMakeFunction( - Lookup lookup, MethodHandle handle, Class functionInterface) { - try { - return makeFunction(lookup, handle, functionInterface); - } catch (Throwable e) { - ExceptionUtils.ignore(e); - throw new IllegalStateException(); - } - } - - private static final MethodType jdkFunctionMethodType = - MethodType.methodType(Object.class, Object.class); - - @SuppressWarnings("unchecked") - public static Function makeJDKFunction(Lookup lookup, MethodHandle handle) { - return makeJDKFunction(lookup, handle, jdkFunctionMethodType); - } - - @SuppressWarnings("unchecked") - public static Function makeJDKFunction( - Lookup lookup, MethodHandle handle, MethodType methodType) { - try { - CallSite callSite = - LambdaMetafactory.metafactory( - lookup, - "apply", - MethodType.methodType(Function.class), - methodType, - handle, - boxedMethodType(handle.type())); - return (Function) callSite.getTarget().invokeExact(); - } catch (Throwable e) { - throw ExceptionUtils.throwException(e); - } - } - - private static final MethodType jdkConsumerMethodType = - MethodType.methodType(void.class, Object.class); - - @SuppressWarnings("unchecked") - public static Consumer makeJDKConsumer(Lookup lookup, MethodHandle handle) { - try { - CallSite callSite = - LambdaMetafactory.metafactory( - lookup, - "accept", - MethodType.methodType(Consumer.class), - jdkConsumerMethodType, - handle, - boxedMethodType(handle.type())); - return (Consumer) callSite.getTarget().invokeExact(); - } catch (Throwable e) { - throw ExceptionUtils.throwException(e); - } - } - - private static final MethodType jdkBiConsumerMethodType = - MethodType.methodType(void.class, Object.class, Object.class); - - @SuppressWarnings("unchecked") - public static BiConsumer makeJDKBiConsumer(Lookup lookup, MethodHandle handle) { - try { - CallSite callSite = - LambdaMetafactory.metafactory( - lookup, - "accept", - MethodType.methodType(BiConsumer.class), - jdkBiConsumerMethodType, - handle, - boxedMethodType(handle.type())); - return (BiConsumer) callSite.getTarget().invokeExact(); - } catch (Throwable e) { - throw ExceptionUtils.throwException(e); - } - } - - private static MethodType boxedMethodType(MethodType methodType) { - Class[] paramTypes = new Class[methodType.parameterCount()]; - for (int i = 0; i < paramTypes.length; i++) { - Class t = methodType.parameterType(i); - if (t.isPrimitive()) { - t = TypeUtils.wrap(t); - } - paramTypes[i] = t; - } - return MethodType.methodType(methodType.returnType(), paramTypes); - } - - @SuppressWarnings("unchecked") - public static T makeFunction(Lookup lookup, MethodHandle handle, Method methodToImpl) { - MethodType instantiatedMethodType = boxedMethodType(handle.type()); - MethodType methodToImplType = - MethodType.methodType(methodToImpl.getReturnType(), methodToImpl.getParameterTypes()); - try { - CallSite callSite = - LambdaMetafactory.metafactory( - lookup, - methodToImpl.getName(), - MethodType.methodType(methodToImpl.getDeclaringClass()), - methodToImplType, - handle, - instantiatedMethodType); - return (T) callSite.getTarget().invokeExact(); - } catch (Throwable e) { - throw ExceptionUtils.throwException(e); - } - } - - public static T makeFunction(Lookup lookup, MethodHandle handle, Class functionInterface) { - String invokedName = "apply"; - try { - Method method = null; - Method[] methods = functionInterface.getMethods(); - for (Method interfaceMethod : methods) { - if (interfaceMethod.getName().equals(invokedName)) { - method = interfaceMethod; - break; - } - } - if (method == null) { - Preconditions.checkArgument(methods.length == 1); - method = methods[0]; - invokedName = method.getName(); - } - MethodType interfaceType = - MethodType.methodType(method.getReturnType(), method.getParameterTypes()); - CallSite callSite = - LambdaMetafactory.metafactory( - lookup, - invokedName, - MethodType.methodType(functionInterface), - interfaceType, - handle, - interfaceType); - return (T) callSite.getTarget().invoke(); - } catch (Throwable e) { - throw ExceptionUtils.throwException(e); - } - } - - private static final Map, Tuple2, String>> methodMap = new HashMap<>(); - - static { - methodMap.put(boolean.class, Tuple2.of(Predicate.class, "test")); - methodMap.put(byte.class, Tuple2.of(ToByteFunction.class, "applyAsByte")); - methodMap.put(char.class, Tuple2.of(ToCharFunction.class, "applyAsChar")); - methodMap.put(short.class, Tuple2.of(ToShortFunction.class, "applyAsShort")); - methodMap.put(int.class, Tuple2.of(ToIntFunction.class, "applyAsInt")); - methodMap.put(long.class, Tuple2.of(ToLongFunction.class, "applyAsLong")); - methodMap.put(float.class, Tuple2.of(ToFloatFunction.class, "applyAsFloat")); - methodMap.put(double.class, Tuple2.of(ToDoubleFunction.class, "applyAsDouble")); - } - - public static Tuple2, String> getterMethodInfo(Class type) { - Tuple2, String> info = methodMap.get(type); - if (info == null) { - return Tuple2.of(Function.class, "apply"); - } - return info; - } - - public static Object makeGetterFunction( - MethodHandles.Lookup lookup, MethodHandle handle, Class returnType) { - Tuple2, String> methodInfo = methodMap.get(returnType); - MethodType factoryType; - if (methodInfo == null) { - methodInfo = Tuple2.of(Function.class, "apply"); - factoryType = jdkFunctionMethodType; - } else { - factoryType = MethodType.methodType(returnType, Object.class); - } - try { - CallSite callSite = - LambdaMetafactory.metafactory( - lookup, - methodInfo.f1, - MethodType.methodType(methodInfo.f0), - factoryType, - handle, - handle.type()); - return callSite.getTarget().invoke(); - } catch (LambdaConversionException e) { - return makeGetterFallback(handle, returnType); - } catch (ClassNotFoundException | NoClassDefFoundError e) { - return makeGetterFunction(lookup, handle, Object.class); - } catch (Throwable e) { - throw ExceptionUtils.throwException(e); - } - } - - private static Object makeGetterFallback(MethodHandle handle, Class returnType) { - if (returnType == boolean.class) { - return (Predicate) - value -> { - try { - return (boolean) handle.invoke(value); - } catch (Throwable e) { - throw ExceptionUtils.throwException(e); - } - }; - } else if (returnType == byte.class) { - return (ToByteFunction) - value -> { - try { - return (byte) handle.invoke(value); - } catch (Throwable e) { - throw ExceptionUtils.throwException(e); - } - }; - } else if (returnType == char.class) { - return (ToCharFunction) - value -> { - try { - return (char) handle.invoke(value); - } catch (Throwable e) { - throw ExceptionUtils.throwException(e); - } - }; - } else if (returnType == short.class) { - return (ToShortFunction) - value -> { - try { - return (short) handle.invoke(value); - } catch (Throwable e) { - throw ExceptionUtils.throwException(e); - } - }; - } else if (returnType == int.class) { - return (ToIntFunction) - value -> { - try { - return (int) handle.invoke(value); - } catch (Throwable e) { - throw ExceptionUtils.throwException(e); - } - }; - } else if (returnType == long.class) { - return (ToLongFunction) - value -> { - try { - return (long) handle.invoke(value); - } catch (Throwable e) { - throw ExceptionUtils.throwException(e); - } - }; - } else if (returnType == float.class) { - return (ToFloatFunction) - value -> { - try { - return (float) handle.invoke(value); - } catch (Throwable e) { - throw ExceptionUtils.throwException(e); - } - }; - } else if (returnType == double.class) { - return (ToDoubleFunction) - value -> { - try { - return (double) handle.invoke(value); - } catch (Throwable e) { - throw ExceptionUtils.throwException(e); - } - }; - } - return (Function) - value -> { - try { - return handle.invoke(value); - } catch (Throwable e) { - throw ExceptionUtils.throwException(e); - } - }; - } - - public static Object getModule(Class cls) { - Preconditions.checkArgument(JdkVersion.MAJOR_VERSION >= 9); - return cls.getModule(); - } - - public static Object addReads(Object thisModule, Object otherModule) { - Preconditions.checkArgument(JdkVersion.MAJOR_VERSION >= 9); - return ((Module) thisModule).addReads((Module) otherModule); - } - - public static Lookup privateLookupIn(Class targetClass, Lookup caller) { - return _Lookup.privateLookupIn(targetClass, caller); - } -} diff --git a/java/fory-core/src/main/java25/org/apache/fory/serializer/PlatformStringUtils.java b/java/fory-core/src/main/java25/org/apache/fory/serializer/PlatformStringUtils.java index e5ac8063cb..4eef97442b 100644 --- a/java/fory-core/src/main/java25/org/apache/fory/serializer/PlatformStringUtils.java +++ b/java/fory-core/src/main/java25/org/apache/fory/serializer/PlatformStringUtils.java @@ -20,30 +20,32 @@ package org.apache.fory.serializer; import java.lang.invoke.MethodHandles; +import java.lang.invoke.MethodHandles.Lookup; import java.lang.invoke.VarHandle; +import java.lang.reflect.Field; import java.nio.ByteOrder; -import java.nio.charset.StandardCharsets; import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.memory.NativeByteOrder; -import org.apache.fory.platform.AndroidSupport; import org.apache.fory.platform.GraalvmSupport; import org.apache.fory.platform.internal._JDKAccess; +import org.apache.fory.util.Preconditions; /** JDK25 string internals used by {@link StringSerializer}. */ final class PlatformStringUtils { - static final boolean JDK_STRING_FIELD_ACCESS = - !AndroidSupport.IS_ANDROID - && !GraalvmSupport.IN_GRAALVM_NATIVE_IMAGE - && _JDKAccess.JDK_STRING_FIELD_ACCESS; + private static final StringHandles STRING_HANDLES = stringHandles(); + + static final boolean JDK_STRING_FIELD_ACCESS = STRING_HANDLES.fieldAccess; static final boolean STRING_VALUE_FIELD_IS_CHARS = - JDK_STRING_FIELD_ACCESS && _JDKAccess.STRING_VALUE_FIELD_IS_CHARS; + JDK_STRING_FIELD_ACCESS && STRING_HANDLES.valueFieldIsChars; static final boolean STRING_VALUE_FIELD_IS_BYTES = - JDK_STRING_FIELD_ACCESS && _JDKAccess.STRING_VALUE_FIELD_IS_BYTES; - static final boolean STRING_HAS_COUNT_OFFSET = - JDK_STRING_FIELD_ACCESS && _JDKAccess.STRING_HAS_COUNT_OFFSET; + JDK_STRING_FIELD_ACCESS && STRING_HANDLES.valueFieldIsBytes; + static final boolean STRING_HAS_COUNT_OFFSET = JDK_STRING_FIELD_ACCESS && STRING_HANDLES.counted; + + private static final VarHandle STRING_VALUE_HANDLE = STRING_HANDLES.value; + private static final VarHandle STRING_CODER_HANDLE = STRING_HANDLES.coder; + private static final VarHandle STRING_COUNT_HANDLE = STRING_HANDLES.count; + private static final VarHandle STRING_OFFSET_HANDLE = STRING_HANDLES.offset; - private static final byte LATIN1 = 0; - private static final byte UTF16 = 1; private static final VarHandle BYTE_ARRAY_LONG = MethodHandles.byteArrayViewVarHandle(long[].class, ByteOrder.nativeOrder()); private static final VarHandle BYTE_ARRAY_CHAR = @@ -51,47 +53,127 @@ final class PlatformStringUtils { private PlatformStringUtils() {} - static Object getStringValue(String value) { - return _JDKAccess.getStringValue(value); + private static StringHandles stringHandles() { + if (GraalvmSupport.IN_GRAALVM_NATIVE_IMAGE || !_JDKAccess.JDK_INTERNAL_FIELD_ACCESS) { + return StringHandles.noAccess(); + } + try { + Field valueField = String.class.getDeclaredField("value"); + boolean valueFieldIsChars = valueField.getType() == char[].class; + boolean valueFieldIsBytes = valueField.getType() == byte[].class; + Field countField = getStringFieldNullable("count"); + Field offsetField = getStringFieldNullable("offset"); + boolean counted = false; + if (countField != null || offsetField != null) { + Preconditions.checkArgument( + countField != null && offsetField != null, "Current jdk not supported"); + Preconditions.checkArgument( + countField.getType() == int.class && offsetField.getType() == int.class, + "Current jdk not supported"); + counted = true; + } + try { + Lookup stringLookup = _JDKAccess._trustedLookup(String.class); + return new StringHandles( + true, + valueFieldIsChars, + valueFieldIsBytes, + counted, + stringLookup.findVarHandle(String.class, "value", valueField.getType()), + valueFieldIsBytes + ? stringLookup.findVarHandle(String.class, "coder", byte.class) + : null, + countField == null + ? null + : stringLookup.findVarHandle(String.class, "count", int.class), + offsetField == null + ? null + : stringLookup.findVarHandle(String.class, "offset", int.class)); + } catch (Throwable ignored) { + return StringHandles.noAccess(); + } + } catch (NoSuchFieldException e) { + throw new RuntimeException(e); + } } - static byte getStringCoder(String value) { - return _JDKAccess.getStringCoder(value); + private static Field getStringFieldNullable(String fieldName) { + try { + return String.class.getDeclaredField(fieldName); + } catch (NoSuchFieldException e) { + return null; + } } - static int getStringOffset(String value) { - return _JDKAccess.getStringOffset(value); + private static final class StringHandles { + private final boolean fieldAccess; + private final boolean valueFieldIsChars; + private final boolean valueFieldIsBytes; + private final boolean counted; + private final VarHandle value; + private final VarHandle coder; + private final VarHandle count; + private final VarHandle offset; + + private StringHandles( + boolean fieldAccess, + boolean valueFieldIsChars, + boolean valueFieldIsBytes, + boolean counted, + VarHandle value, + VarHandle coder, + VarHandle count, + VarHandle offset) { + this.fieldAccess = fieldAccess; + this.valueFieldIsChars = valueFieldIsChars; + this.valueFieldIsBytes = valueFieldIsBytes; + this.counted = counted; + this.value = value; + this.coder = coder; + this.count = count; + this.offset = offset; + } + + private static StringHandles noAccess() { + return new StringHandles(false, false, false, false, null, null, null, null); + } } - static int getStringCount(String value) { - return _JDKAccess.getStringCount(value); + static Object getStringValue(String value) { + checkStringAccess("String.value"); + return STRING_VALUE_HANDLE.get(value); } - static String newCharsStringZeroCopy(char[] data) { - if (!JDK_STRING_FIELD_ACCESS) { - return new String(data); + static byte getStringCoder(String value) { + checkStringAccess("String.coder"); + if (STRING_CODER_HANDLE == null) { + throw new UnsupportedOperationException("String.coder is not available on this JDK"); } - return _JDKAccess.newCharsStringZeroCopy(data); + return (byte) STRING_CODER_HANDLE.get(value); } - static String newBytesStringZeroCopy(byte coder, byte[] data) { - if (!JDK_STRING_FIELD_ACCESS) { - return newBytesStringSlow(coder, data); + static int getStringOffset(String value) { + checkStringAccess("String.offset"); + if (STRING_OFFSET_HANDLE == null) { + throw new UnsupportedOperationException("String.offset is not available on this JDK"); } - return _JDKAccess.newBytesStringZeroCopy(coder, data); + return (int) STRING_OFFSET_HANDLE.get(value); } - private static String newBytesStringSlow(byte coder, byte[] data) { - if (coder == LATIN1) { - return new String(data, StandardCharsets.ISO_8859_1); - } else if (coder == UTF16) { - char[] chars = new char[data.length >> 1]; - for (int i = 0, j = 0; i < data.length; i += 2) { - chars[j++] = (char) ((data[i] & 0xff) | ((data[i + 1] & 0xff) << 8)); - } - return new String(chars); - } else { - return new String(data, StandardCharsets.UTF_8); + static int getStringCount(String value) { + checkStringAccess("String.count"); + if (STRING_COUNT_HANDLE == null) { + throw new UnsupportedOperationException("String.count is not available on this JDK"); + } + return (int) STRING_COUNT_HANDLE.get(value); + } + + private static void checkStringAccess(String target) { + if (!JDK_STRING_FIELD_ACCESS) { + throw new UnsupportedOperationException( + target + + " private access is unavailable; open java.base/java.lang.invoke to " + + "org.apache.fory.core"); } } diff --git a/java/fory-core/src/main/java25/org/apache/fory/serializer/SerializationHookLookup.java b/java/fory-core/src/main/java25/org/apache/fory/serializer/SerializationHookLookup.java new file mode 100644 index 0000000000..f94aa161fc --- /dev/null +++ b/java/fory-core/src/main/java25/org/apache/fory/serializer/SerializationHookLookup.java @@ -0,0 +1,143 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.serializer; + +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.lang.invoke.MethodHandle; +import java.lang.invoke.MethodHandles; +import java.lang.invoke.SerializedLambda; +import java.lang.reflect.Method; +import java.lang.reflect.Modifier; +import org.apache.fory.collection.ClassValueCache; +import org.apache.fory.platform.internal._JDKAccess; +import org.apache.fory.util.ExceptionUtils; + +final class SerializationHookLookup { + private SerializationHookLookup() {} + + static MethodHandle readResolveHandle(Class type, Method method) throws IllegalAccessException { + try { + return _JDKAccess._trustedLookup(type).unreflect(method); + } catch (IllegalArgumentException e) { + if (type != SerializedLambda.class) { + throw e; + } + // JDK25 rejects SerializedLambda itself as a privateLookupIn target. Reflective access still + // honors the same java.base/java.lang.invoke open requirement and preserves the serializer + // path used by JDK8-24. + try { + method.setAccessible(true); + } catch (RuntimeException inaccessible) { + throw new IllegalStateException( + "SerializedLambda readResolve requires java.base/java.lang.invoke to be open to " + + "org.apache.fory.core", + inaccessible); + } + return MethodHandles.lookup().unreflect(method); + } + } + + private static final ClassValueCache methodsCache = ClassValueCache.newClassKeyCache(32); + + private static final class Methods { + private final Method writeObject; + private final Method readObject; + private final Method readObjectNoData; + private final Method writeReplace; + private final Method readResolve; + + private Methods(Class type) { + writeObject = getPrivateMethod(type, "writeObject", void.class, ObjectOutputStream.class); + readObject = getPrivateMethod(type, "readObject", void.class, ObjectInputStream.class); + readObjectNoData = getPrivateMethod(type, "readObjectNoData", void.class); + writeReplace = getInheritableObjectMethod(type, "writeReplace"); + readResolve = getInheritableObjectMethod(type, "readResolve"); + } + } + + private static Methods methods(Class type) { + return methodsCache.get(type, () -> new Methods(type)); + } + + private static Method getPrivateMethod( + Class type, String methodName, Class returnType, Class... parameterTypes) { + try { + Method method = type.getDeclaredMethod(methodName, parameterTypes); + int modifiers = method.getModifiers(); + if (Modifier.isPrivate(modifiers) + && !Modifier.isStatic(modifiers) + && method.getReturnType() == returnType) { + return method; + } + } catch (NoSuchMethodException | SecurityException e) { + ExceptionUtils.ignore(e); + } + return null; + } + + private static Method getInheritableObjectMethod(Class type, String methodName) { + Class cls = type; + while (cls != null) { + try { + Method method = cls.getDeclaredMethod(methodName); + int modifiers = method.getModifiers(); + if (!Modifier.isStatic(modifiers) && method.getReturnType() == Object.class) { + return method; + } + return null; + } catch (NoSuchMethodException e) { + ExceptionUtils.ignore(e); + } catch (SecurityException e) { + return null; + } + cls = cls.getSuperclass(); + } + return null; + } + + static Method getWriteObjectMethod(Class type) { + return methods(type).writeObject; + } + + static Method getReadObjectMethod(Class type) { + return methods(type).readObject; + } + + static Method getReadObjectNoDataMethod(Class type) { + return methods(type).readObjectNoData; + } + + static MethodHandle getDefaultReadObjectHandle(Class type) { + return null; + } + + static Method getWriteReplaceMethod(Class type) { + return methods(type).writeReplace; + } + + static Method getReadResolveMethod(Class type) { + return methods(type).readResolve; + } + + static boolean isAvailable() { + return true; + } +} diff --git a/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties b/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties index 71837d675c..69fb5bfd0e 100644 --- a/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties +++ b/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties @@ -538,7 +538,7 @@ Args=--initialize-at-build-time=org.apache.fory.annotation.ForyField$Dynamic,\ org.apache.fory.util.record.RecordUtils$2,\ org.apache.fory.util.record.RecordUtils$3,\ org.apache.fory.util.record.RecordUtils,\ - org.apache.fory.platform.internal._JDKAccess$StringCoderField,\ + org.apache.fory.serializer.PlatformStringUtils$StringCoderField,\ org.apache.fory.platform.internal._JDKAccess,\ org.apache.fory.platform.internal._Lookup,\ org.apache.fory.codegen.JaninoUtils$CodeStats,\ diff --git a/java/fory-core/src/test/java/org/apache/fory/serializer/StringSerializerTest.java b/java/fory-core/src/test/java/org/apache/fory/serializer/StringSerializerTest.java index 7765262bf6..efe625cc89 100644 --- a/java/fory-core/src/test/java/org/apache/fory/serializer/StringSerializerTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/serializer/StringSerializerTest.java @@ -37,7 +37,6 @@ import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.memory.MemoryUtils; import org.apache.fory.platform.JdkVersion; -import org.apache.fory.platform.internal._JDKAccess; import org.apache.fory.util.MathUtils; import org.apache.fory.util.StringUtils; import org.testng.Assert; @@ -138,7 +137,7 @@ private static boolean writeJavaStringZeroCopy(MemoryBuffer buffer, String value } static void writeJDK8String(MemoryBuffer buffer, String value) { - final char[] chars = (char[]) _JDKAccess.getStringValue(value); + final char[] chars = (char[]) PlatformStringUtils.getStringValue(value); int numBytes = MathUtils.doubleExact(value.length()); buffer.writeCharsWithSize(chars); } diff --git a/java/fory-core/src/test/java/org/apache/fory/serializer/collection/CollectionSerializersTest.java b/java/fory-core/src/test/java/org/apache/fory/serializer/collection/CollectionSerializersTest.java index 7f6e648f79..ed762c7b3d 100644 --- a/java/fory-core/src/test/java/org/apache/fory/serializer/collection/CollectionSerializersTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/serializer/collection/CollectionSerializersTest.java @@ -73,6 +73,7 @@ import org.apache.fory.annotation.ForyConstructor; import org.apache.fory.context.ReadContext; import org.apache.fory.exception.DeserializationException; +import org.apache.fory.exception.SerializationException; import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.memory.MemoryUtils; import org.apache.fory.reflect.TypeRef; @@ -99,6 +100,24 @@ public int compare(String left, String right) { } } + public static final class NonComparableSetItem { + public int id; + + public NonComparableSetItem() {} + + private NonComparableSetItem(int id) { + this.id = id; + } + } + + public static final class SetItemComparator + implements Comparator, Serializable { + @Override + public int compare(NonComparableSetItem left, NonComparableSetItem right) { + return Integer.compare(left.id, right.id); + } + } + private static final class SortedSetConstructorCase { private final String name; private final Class expectedType; @@ -290,6 +309,66 @@ public void testSortedSet(Fory fory) { Assert.assertEquals(Arrays.toString(copy.toArray()), "[str, str2, str11]"); } + @Test + public void testXlangTreeSetCopy() { + Fory fory = + builder().withXlang(true).withRefTracking(false).requireClassRegistration(true).build(); + TreeSet set = new TreeSet<>(Collections.reverseOrder()); + set.addAll(sortedCollectionInput()); + + Assert.assertTrue( + fory.getSerializer(TreeSet.class) instanceof CollectionSerializers.SortedSetSerializer); + TreeSet copy = fory.copy(set); + + Assert.assertNotSame(copy, set); + Assert.assertEquals(copy, set); + Assert.assertNotNull(copy.comparator()); + Assert.assertTrue(copy.comparator().compare("a", "b") > 0); + } + + @Test + public void testXlangTreeSetNaturalComparator() { + Fory fory = + builder().withXlang(true).withRefTracking(false).requireClassRegistration(true).build(); + TreeSet set = new TreeSet<>(Comparator.naturalOrder()); + set.addAll(sortedCollectionInput()); + + Object decoded = fory.deserialize(fory.serialize(set)); + + Assert.assertEquals(decoded, set); + } + + @Test + public void testXlangTreeSetComparatorWrite() { + Fory fory = + builder().withXlang(true).withRefTracking(false).requireClassRegistration(true).build(); + TreeSet set = new TreeSet<>(new SetItemComparator()); + set.add(new NonComparableSetItem(1)); + + SerializationException exception = + Assert.expectThrows(SerializationException.class, () -> fory.serialize(set)); + Assert.assertTrue(exception.getCause() instanceof UnsupportedOperationException); + } + + @Test + public void testXlangTreeSetObjectCopy() { + Fory fory = + builder().withXlang(true).withRefTracking(false).requireClassRegistration(true).build(); + fory.register(NonComparableSetItem.class); + fory.register(SetItemComparator.class); + TreeSet set = new TreeSet<>(new SetItemComparator()); + set.add(new NonComparableSetItem(2)); + set.add(new NonComparableSetItem(1)); + + TreeSet copy = fory.copy(set); + + Assert.assertNotSame(copy, set); + Assert.assertEquals(copy.size(), set.size()); + Assert.assertNotNull(copy.comparator()); + Assert.assertEquals(copy.first().id, 1); + Assert.assertEquals(copy.last().id, 2); + } + private class TestComparator implements Comparator { AtomicInteger i; @@ -344,7 +423,7 @@ public void testTreeSetConstructorMatrix(Fory fory) { } @Test(dataProvider = "referenceTrackingConfig") - public void testConcurrentSkipListSetConstructorMatrix(boolean referenceTrackingConfig) { + public void testSkipListSetCtorSerde(boolean referenceTrackingConfig) { Fory fory = Fory.builder() .withXlang(false) @@ -362,7 +441,7 @@ public void testConcurrentSkipListSetConstructorMatrix(boolean referenceTracking } @Test(dataProvider = "foryCopyConfig") - public void testConcurrentSkipListSetConstructorMatrix(Fory fory) { + public void testSkipListSetCtorCopy(Fory fory) { for (SortedSetConstructorCase testCase : concurrentSkipListSetConstructorCases()) { SortedSet original = testCase.factory.get(); assertSortedSetState(testCase, original); @@ -423,7 +502,7 @@ public ChildTreeSetWithComparator(Comparator comparator) { } @Test(dataProvider = "referenceTrackingConfig") - public void testSortedSetSubclassWithoutComparatorCtor(boolean referenceTrackingConfig) { + public void testSortedSetSubclassNoComparatorCtor(boolean referenceTrackingConfig) { Fory fory = Fory.builder() .withXlang(false) @@ -457,8 +536,7 @@ public void testSortedSetSubclassWithComparatorCtor(boolean referenceTrackingCon } @Test(dataProvider = "referenceTrackingConfig") - public void testSortedSetSubclassRegisteredWithSortedSetSerializer( - boolean referenceTrackingConfig) { + public void testSortedSetSubclassRegistered(boolean referenceTrackingConfig) { Fory fory = Fory.builder() .withXlang(false) @@ -481,8 +559,7 @@ public void testSortedSetSubclassRegisteredWithSortedSetSerializer( } @Test(dataProvider = "referenceTrackingConfig") - public void testSortedSetSubclassWithComparatorRegisteredWithSortedSetSerializer( - boolean referenceTrackingConfig) { + public void testSortedSetComparatorRegistered(boolean referenceTrackingConfig) { Fory fory = Fory.builder() .withXlang(false) @@ -918,7 +995,7 @@ public void testSetFromMap(Fory fory) { } @Test - public void testSetFromMapNestedInExternalizablePreservesRefIds() { + public void testSetFromMapExternalizableRefs() { Fory fory = Fory.builder() .withXlang(false) @@ -1047,7 +1124,7 @@ public void testSerializeJavaBlockingQueue() { } @Test - public void testDeserializeJavaBlockingQueueRejectsMalformedCapacity() { + public void testBlockingQueueBadCapacity() { Fory fory = Fory.builder() .withXlang(false) @@ -1078,7 +1155,7 @@ public void testDeserializeJavaBlockingQueueRejectsMalformedCapacity() { } @Test - public void testCollectionReadRejectsOversizedElementCount() { + public void testCollectionRejectsTooManyElements() { Fory fory = Fory.builder() .withXlang(false) @@ -1096,7 +1173,7 @@ public void testCollectionReadRejectsOversizedElementCount() { } @Test - public void testBitSetReadRejectsNegativeDecodedBinaryPayload() { + public void testBitSetRejectsNegativeBinary() { Fory fory = Fory.builder().withXlang(false).build(); MemoryBuffer buffer = MemoryBuffer.newHeapBuffer(5); writeNegativeDecodedVarUInt32(buffer); @@ -1438,7 +1515,7 @@ public void testDefaultCollectionSerializer(Fory fory) { } @Test - public void testDefaultCollectionSerializerAsyncCompilation() { + public void testDefaultCollectionAsyncCompile() { Fory fory = Fory.builder() .withXlang(false) @@ -1564,7 +1641,7 @@ static class CollectionAbstractTest { } @Test(dataProvider = "enableCodegen") - public void testAbstractCollectionElementsSerialization(boolean enableCodegen) { + public void testAbstractCollectionElementsSerde(boolean enableCodegen) { Fory fory = Fory.builder() .withXlang(false) @@ -1584,7 +1661,7 @@ public void testAbstractCollectionElementsSerialization(boolean enableCodegen) { } @Test(dataProvider = "foryCopyConfig") - public void testAbstractCollectionElementsSerialization(Fory fory) { + public void testAbstractCollectionElementsCopy(Fory fory) { { CollectionAbstractTest test = new CollectionAbstractTest(); test.fooList = new ArrayList<>(ImmutableList.of(new Foo1(), new Foo1())); diff --git a/java/fory-core/src/test/java/org/apache/fory/serializer/collection/MapSerializersTest.java b/java/fory-core/src/test/java/org/apache/fory/serializer/collection/MapSerializersTest.java index dba05453e5..c4206e7397 100644 --- a/java/fory-core/src/test/java/org/apache/fory/serializer/collection/MapSerializersTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/serializer/collection/MapSerializersTest.java @@ -59,6 +59,7 @@ import org.apache.fory.collection.LazyMap; import org.apache.fory.collection.MapEntry; import org.apache.fory.exception.DeserializationException; +import org.apache.fory.exception.SerializationException; import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.memory.MemoryUtils; import org.apache.fory.reflect.TypeRef; @@ -85,6 +86,24 @@ public int compare(String left, String right) { } } + public static final class NonComparableMapKey { + public int id; + + public NonComparableMapKey() {} + + private NonComparableMapKey(int id) { + this.id = id; + } + } + + public static final class MapKeyComparator + implements Comparator, Serializable { + @Override + public int compare(NonComparableMapKey left, NonComparableMapKey right) { + return Integer.compare(left.id, right.id); + } + } + private static final class SortedMapConstructorCase { private final String name; private final Class expectedType; @@ -398,6 +417,66 @@ public void testTreeMap(Fory fory) { copyCheck(fory, beanForMap); } + @Test + public void testXlangTreeMapCopy() { + Fory fory = + builder().withXlang(true).withRefTracking(false).requireClassRegistration(true).build(); + TreeMap map = new TreeMap<>(Collections.reverseOrder()); + map.putAll(sortedMapInput()); + + Assert.assertTrue( + fory.getSerializer(TreeMap.class) instanceof MapSerializers.SortedMapSerializer); + TreeMap copy = fory.copy(map); + + Assert.assertNotSame(copy, map); + Assert.assertEquals(copy, map); + Assert.assertNotNull(copy.comparator()); + Assert.assertTrue(copy.comparator().compare("a", "b") > 0); + } + + @Test + public void testXlangTreeMapNaturalComparator() { + Fory fory = + builder().withXlang(true).withRefTracking(false).requireClassRegistration(true).build(); + TreeMap map = new TreeMap<>(Comparator.naturalOrder()); + map.putAll(sortedMapInput()); + + Object decoded = fory.deserialize(fory.serialize(map)); + + Assert.assertEquals(decoded, map); + } + + @Test + public void testXlangTreeMapComparatorWrite() { + Fory fory = + builder().withXlang(true).withRefTracking(false).requireClassRegistration(true).build(); + TreeMap map = new TreeMap<>(new MapKeyComparator()); + map.put(new NonComparableMapKey(1), 1); + + SerializationException exception = + Assert.expectThrows(SerializationException.class, () -> fory.serialize(map)); + Assert.assertTrue(exception.getCause() instanceof UnsupportedOperationException); + } + + @Test + public void testXlangTreeMapObjectCopy() { + Fory fory = + builder().withXlang(true).withRefTracking(false).requireClassRegistration(true).build(); + fory.register(NonComparableMapKey.class); + fory.register(MapKeyComparator.class); + TreeMap map = new TreeMap<>(new MapKeyComparator()); + map.put(new NonComparableMapKey(2), 2); + map.put(new NonComparableMapKey(1), 1); + + TreeMap copy = fory.copy(map); + + Assert.assertNotSame(copy, map); + Assert.assertEquals(copy.size(), map.size()); + Assert.assertNotNull(copy.comparator()); + Assert.assertEquals(copy.firstKey().id, 1); + Assert.assertEquals(copy.lastKey().id, 2); + } + @Test(dataProvider = "referenceTrackingConfig") public void testTreeMapConstructorMatrix(boolean referenceTrackingConfig) { Fory fory = @@ -429,7 +508,7 @@ public void testTreeMapConstructorMatrix(Fory fory) { } @Test(dataProvider = "referenceTrackingConfig") - public void testConcurrentSkipListMapConstructorMatrix(boolean referenceTrackingConfig) { + public void testSkipListMapCtorSerde(boolean referenceTrackingConfig) { Fory fory = builder() .withXlang(false) @@ -447,7 +526,7 @@ public void testConcurrentSkipListMapConstructorMatrix(boolean referenceTracking } @Test(dataProvider = "foryCopyConfig") - public void testConcurrentSkipListMapConstructorMatrix(Fory fory) { + public void testSkipListMapCtorCopy(Fory fory) { for (SortedMapConstructorCase testCase : concurrentSkipListMapConstructorCases()) { SortedMap original = testCase.factory.get(); assertSortedMapState(testCase, original); @@ -477,7 +556,7 @@ public ChildTreeMapWithComparator(Comparator comparator) { } @Test(dataProvider = "referenceTrackingConfig") - public void testSortedMapSubclassWithoutComparatorCtor(boolean referenceTrackingConfig) { + public void testSortedMapSubclassNoComparatorCtor(boolean referenceTrackingConfig) { Fory fory = builder() .withXlang(false) @@ -511,8 +590,7 @@ public void testSortedMapSubclassWithComparatorCtor(boolean referenceTrackingCon } @Test(dataProvider = "referenceTrackingConfig") - public void testSortedMapSubclassRegisteredWithSortedMapSerializer( - boolean referenceTrackingConfig) { + public void testSortedMapSubclassRegistered(boolean referenceTrackingConfig) { Fory fory = builder() .withXlang(false) @@ -532,8 +610,7 @@ public void testSortedMapSubclassRegisteredWithSortedMapSerializer( } @Test(dataProvider = "referenceTrackingConfig") - public void testSortedMapSubclassWithComparatorRegisteredWithSortedMapSerializer( - boolean referenceTrackingConfig) { + public void testSortedMapComparatorRegistered(boolean referenceTrackingConfig) { Fory fory = builder() .withXlang(false) @@ -1072,7 +1149,7 @@ public void testStringKeyMapSerializer() { } @Test(dataProvider = "enableCodegen") - public void testMapElementRefOverrideReadRespectsHeader(boolean enableCodegen) { + public void testMapElementRefOverrideHeader(boolean enableCodegen) { Fory foryNoRef = builder() .withXlang(false) diff --git a/kotlin/fory-kotlin-ksp/src/main/kotlin/org/apache/fory/kotlin/ksp/KotlinSerializerSourceWriter.kt b/kotlin/fory-kotlin-ksp/src/main/kotlin/org/apache/fory/kotlin/ksp/KotlinSerializerSourceWriter.kt index 93cf8a06da..dd33bb4b8d 100644 --- a/kotlin/fory-kotlin-ksp/src/main/kotlin/org/apache/fory/kotlin/ksp/KotlinSerializerSourceWriter.kt +++ b/kotlin/fory-kotlin-ksp/src/main/kotlin/org/apache/fory/kotlin/ksp/KotlinSerializerSourceWriter.kt @@ -26,6 +26,9 @@ import java.nio.charset.StandardCharsets internal class KotlinSerializerSourceWriter(private val struct: KotlinSourceStruct) { private val builder = StringBuilder(32768) + private val hasComparatorGuards = struct.fields.any { needsComparatorGuard(it.type) } + + private data class ContainerTarget(val init: String, val result: String) fun writeTo(codeGenerator: CodeGenerator) { val dependencies = @@ -128,6 +131,12 @@ internal class KotlinSerializerSourceWriter(private val struct: KotlinSourceStru } builder.append(" return Collections.unmodifiableList(descriptors)\n") builder.append(" }\n") + if (hasComparatorGuards) { + builder.append(" private val NATURAL_ORDER_COMPARATOR: java.util.Comparator<*> =\n") + builder.append( + " java.util.Comparator.naturalOrder>() as java.util.Comparator<*>\n" + ) + } builder.append(" }\n\n") builder.append(" override fun getGeneratedDescriptors(): List = DESCRIPTORS\n\n") } @@ -224,6 +233,7 @@ internal class KotlinSerializerSourceWriter(private val struct: KotlinSourceStru .append(" override fun write(writeContext: WriteContext, value: ") .append(struct.typeName) .append(") {\n") + writeComparatorPreflight() builder.append(" val buffer = writeContext.buffer\n") builder.append(" if (typeResolver.checkClassVersion()) {\n") builder.append(" buffer.writeInt32(classVersionHash)\n") @@ -249,6 +259,19 @@ internal class KotlinSerializerSourceWriter(private val struct: KotlinSourceStru builder.append(" }\n") builder.append(" }\n") builder.append(" }\n\n") + if (hasComparatorGuards) { + builder.append( + " private fun requireXlangNaturalOrdering(typeName: String, comparator: java.util.Comparator<*>?) {\n" + ) + builder.append(" if (comparator != null && comparator !== NATURAL_ORDER_COMPARATOR) {\n") + builder.append(" throw UnsupportedOperationException(\n") + builder.append( + " \"Xlang serialization of \$typeName with a custom comparator is unsupported because the xlang container wire format does not encode comparators\"\n" + ) + builder.append(" )\n") + builder.append(" }\n") + builder.append(" }\n\n") + } builder .append(" private fun readCompatibleConstructor(readContext: ReadContext): ") @@ -558,6 +581,191 @@ internal class KotlinSerializerSourceWriter(private val struct: KotlinSourceStru builder.append(" }\n\n") } + private fun writeComparatorPreflight() { + if (!hasComparatorGuards) { + return + } + for (field in struct.fields) { + appendComparatorGuard( + field.type, + "value.${field.name}", + " ", + "${struct.qualifiedTypeName}.${field.name}", + "guard${field.id}", + 0, + ) + } + } + + private fun needsComparatorGuard(type: KotlinSourceTypeNode): Boolean = + when { + type.nullable -> needsComparatorGuard(type.copy(nullable = false)) + type.collectionFactory == CollectionFactory.TREE_SET || + type.collectionFactory == CollectionFactory.CONCURRENT_SKIP_LIST_SET || + type.collectionFactory == CollectionFactory.TREE_MAP || + type.collectionFactory == CollectionFactory.CONCURRENT_SKIP_LIST_MAP -> true + type.typeId == "Types.LIST" || type.typeId == "Types.SET" -> + type.typeArguments.any { needsComparatorGuard(it) } + type.typeId == "Types.MAP" -> type.typeArguments.any { needsComparatorGuard(it) } + else -> false + } + + private fun appendComparatorGuard( + type: KotlinSourceTypeNode, + expression: String, + indent: String, + path: String, + prefix: String, + depth: Int, + ) { + if (!needsComparatorGuard(type)) { + return + } + if (type.nullable) { + val nullableValue = "${prefix}Value$depth" + builder + .append(indent) + .append("val ") + .append(nullableValue) + .append(" = ") + .append(expression) + .append("\n") + builder.append(indent).append("if (").append(nullableValue).append(" != null) {\n") + appendComparatorGuard( + type.copy(nullable = false), + nullableValue, + "$indent ", + path, + prefix, + depth + 1, + ) + builder.append(indent).append("}\n") + return + } + when (type.collectionFactory) { + CollectionFactory.TREE_SET, + CollectionFactory.CONCURRENT_SKIP_LIST_SET -> + builder + .append(indent) + .append("requireXlangNaturalOrdering(\"") + .append(escape(path)) + .append("\", (") + .append(expression) + .append(" as java.util.SortedSet<*>).comparator())\n") + CollectionFactory.TREE_MAP, + CollectionFactory.CONCURRENT_SKIP_LIST_MAP -> + builder + .append(indent) + .append("requireXlangNaturalOrdering(\"") + .append(escape(path)) + .append("\", (") + .append(expression) + .append(" as java.util.SortedMap<*, *>).comparator())\n") + else -> {} + } + when (type.typeId) { + "Types.LIST" -> { + val element = type.typeArguments[0] + if (!needsComparatorGuard(element)) { + return + } + val source = "${prefix}List$depth" + val index = "${prefix}Index$depth" + val elementName = "${prefix}Element$depth" + builder + .append(indent) + .append("val ") + .append(source) + .append(" = ") + .append(expression) + .append("\n") + builder + .append(indent) + .append("if (") + .append(source) + .append(" is java.util.RandomAccess) {\n") + builder.append(indent).append(" var ").append(index).append(" = 0\n") + builder + .append(indent) + .append(" ") + .append("while (") + .append(index) + .append(" < ") + .append(source) + .append(".size) {\n") + appendComparatorGuard( + element, + "$source[$index]", + "$indent ", + "$path element", + prefix, + depth + 1, + ) + builder.append(indent).append(" ").append(index).append("++\n") + builder.append(indent).append(" }\n") + builder.append(indent).append("} else {\n") + builder + .append(indent) + .append(" for (") + .append(elementName) + .append(" in ") + .append(source) + .append(") {\n") + appendComparatorGuard( + element, + elementName, + "$indent ", + "$path element", + prefix, + depth + 1, + ) + builder.append(indent).append(" }\n") + builder.append(indent).append("}\n") + } + "Types.SET" -> { + val element = type.typeArguments[0] + if (!needsComparatorGuard(element)) { + return + } + val elementName = "${prefix}Element$depth" + builder + .append(indent) + .append("for (") + .append(elementName) + .append(" in ") + .append(expression) + .append(") {\n") + appendComparatorGuard( + element, + elementName, + "$indent ", + "$path element", + prefix, + depth + 1, + ) + builder.append(indent).append("}\n") + } + "Types.MAP" -> { + val key = type.typeArguments[0] + val value = type.typeArguments[1] + if (!needsComparatorGuard(key) && !needsComparatorGuard(value)) { + return + } + val entry = "${prefix}Entry$depth" + builder + .append(indent) + .append("for (") + .append(entry) + .append(" in ") + .append(expression) + .append(".entries) {\n") + appendComparatorGuard(key, "$entry.key", "$indent ", "$path key", prefix, depth + 1) + appendComparatorGuard(value, "$entry.value", "$indent ", "$path value", prefix, depth + 2) + builder.append(indent).append("}\n") + } + } + } + private fun writeMutableReadBody() { builder.append(" val value = ").append(struct.typeName).append("()\n") builder.append(" if (readContext.hasPreservedRefId()) {\n") @@ -811,7 +1019,7 @@ internal class KotlinSerializerSourceWriter(private val struct: KotlinSourceStru return } for (field in struct.fields) { - if (field.type.primitive || isScalarUnsigned(field)) { + if (isDirectCopyValue(field.type)) { builder .append(" val ") .append(field.localName) @@ -861,7 +1069,7 @@ internal class KotlinSerializerSourceWriter(private val struct: KotlinSourceStru builder.append(" copyContext.reference(value, copy)\n") for (field in struct.fields) { builder.append(" copy.").append(field.name).append(" = ") - if (field.type.primitive || isScalarUnsigned(field)) { + if (isDirectCopyValue(field.type)) { builder.append("value.").append(field.name).append("\n") } else if (isDenseUnsignedArray(field)) { builder.append("value.").append(field.name) @@ -888,7 +1096,7 @@ internal class KotlinSerializerSourceWriter(private val struct: KotlinSourceStru private fun copyExpression(field: KotlinSourceField): String = when { - field.type.primitive || isScalarUnsigned(field) -> "value.${field.name}" + isDirectCopyValue(field.type) -> "value.${field.name}" isDenseUnsignedArray(field) -> if (field.nullable) "value.${field.name}?.copyOf()" else "value.${field.name}.copyOf()" field.type.isCollectionOrMap() -> @@ -903,7 +1111,7 @@ internal class KotlinSerializerSourceWriter(private val struct: KotlinSourceStru private fun constructorCopyExpression(field: KotlinSourceField): String { val expression = when { - field.type.primitive || isScalarUnsigned(field) -> "value.${field.name}" + isDirectCopyValue(field.type) -> "value.${field.name}" isDenseUnsignedArray(field) -> if (field.nullable) "value.${field.name}?.copyOf()" else "value.${field.name}.copyOf()" field.type.isCollectionOrMap() -> @@ -1023,13 +1231,10 @@ internal class KotlinSerializerSourceWriter(private val struct: KotlinSourceStru val value = "readValue$depth" return "$valueExpression?.let { $value -> ${collectionReadExpression(type.copy(nullable = false), value, depth + 1, erasedInput)} }" } - val adapted = - if (type.typeArguments.any { needsCollectionReadAdaptation(it) }) { - readContainerExpression(type, valueExpression, depth) - } else { - valueExpression - } - return applyCollectionFactory(type, adapted, erasedInput && adapted == valueExpression) + if (type.typeArguments.any { needsCollectionReadAdaptation(it) }) { + return readContainerExpression(type, valueExpression, depth) + } + return applyCollectionFactory(type, valueExpression, erasedInput) } private fun needsCollectionReadAdaptation(type: KotlinSourceTypeNode): Boolean = @@ -1043,23 +1248,25 @@ internal class KotlinSerializerSourceWriter(private val struct: KotlinSourceStru ): String { val source = "readSource$depth" val target = "readTarget$depth" - val valueType = type.valueTypeName.removeSuffix("?") return when (type.typeId) { "Types.LIST" -> { val element = "readElement$depth" val adaptedElement = readElementExpression(type.typeArguments[0], element, depth + 1) - "run { val $source = ${erasedCollectionExpression(type, expression)}; val $target = java.util.ArrayList($source.size); for ($element in $source) { $target.add($adaptedElement) }; $target as $valueType }" + val targetBuild = readTarget(type, source, target) + "run { val $source = ${erasedCollectionExpression(type, expression)}; val $target = ${targetBuild.init}; for ($element in $source) { $target.add($adaptedElement) }; ${targetBuild.result} }" } "Types.SET" -> { val element = "readElement$depth" val adaptedElement = readElementExpression(type.typeArguments[0], element, depth + 1) - "run { val $source = ${erasedCollectionExpression(type, expression)}; val $target = java.util.LinkedHashSet($source.size); for ($element in $source) { $target.add($adaptedElement) }; $target as $valueType }" + val targetBuild = readTarget(type, source, target) + "run { val $source = ${erasedCollectionExpression(type, expression)}; val $target = ${targetBuild.init}; for ($element in $source) { $target.add($adaptedElement) }; ${targetBuild.result} }" } "Types.MAP" -> { val entry = "readEntry$depth" val adaptedKey = readElementExpression(type.typeArguments[0], "$entry.key", depth + 1) val adaptedValue = readElementExpression(type.typeArguments[1], "$entry.value", depth + 2) - "run { val $source = ${erasedCollectionExpression(type, expression)}; val $target = java.util.LinkedHashMap($source.size); for ($entry in $source.entries) { $target[$adaptedKey] = $adaptedValue }; $target as $valueType }" + val targetBuild = readTarget(type, source, target) + "run { val $source = ${erasedCollectionExpression(type, expression)}; val $target = ${targetBuild.init}; for ($entry in $source.entries) { $target[$adaptedKey] = $adaptedValue }; ${targetBuild.result} }" } else -> expression } @@ -1109,6 +1316,61 @@ internal class KotlinSerializerSourceWriter(private val struct: KotlinSourceStru ) } + private fun readTarget( + type: KotlinSourceTypeNode, + source: String, + target: String + ): ContainerTarget { + val valueType = type.valueTypeName.removeSuffix("?") + fun castTarget(): String = "$target as $valueType" + return when (type.typeId) { + "Types.LIST" -> + when (type.collectionFactory) { + CollectionFactory.LINKED_LIST -> + ContainerTarget("java.util.LinkedList()", castTarget()) + CollectionFactory.COPY_ON_WRITE_ARRAY_LIST -> + ContainerTarget( + "java.util.ArrayList($source.size)", + "java.util.concurrent.CopyOnWriteArrayList($target) as $valueType" + ) + else -> ContainerTarget("java.util.ArrayList($source.size)", castTarget()) + } + "Types.SET" -> + when (type.collectionFactory) { + CollectionFactory.HASH_SET -> + ContainerTarget("java.util.HashSet($source.size)", castTarget()) + CollectionFactory.TREE_SET -> ContainerTarget("java.util.TreeSet()", castTarget()) + CollectionFactory.COPY_ON_WRITE_ARRAY_SET -> + ContainerTarget( + "java.util.LinkedHashSet($source.size)", + "java.util.concurrent.CopyOnWriteArraySet($target) as $valueType" + ) + CollectionFactory.CONCURRENT_SKIP_LIST_SET -> + ContainerTarget("java.util.concurrent.ConcurrentSkipListSet()", castTarget()) + else -> ContainerTarget("java.util.LinkedHashSet($source.size)", castTarget()) + } + "Types.MAP" -> + when (type.collectionFactory) { + CollectionFactory.HASH_MAP -> + ContainerTarget("java.util.HashMap($source.size)", castTarget()) + CollectionFactory.TREE_MAP -> + ContainerTarget("java.util.TreeMap()", castTarget()) + CollectionFactory.CONCURRENT_HASH_MAP -> + ContainerTarget( + "java.util.concurrent.ConcurrentHashMap($source.size)", + castTarget() + ) + CollectionFactory.CONCURRENT_SKIP_LIST_MAP -> + ContainerTarget( + "java.util.concurrent.ConcurrentSkipListMap()", + castTarget() + ) + else -> ContainerTarget("java.util.LinkedHashMap($source.size)", castTarget()) + } + else -> ContainerTarget("null", castTarget()) + } + } + private fun localVariableType(field: KotlinSourceField): String { if (field.nullable || field.type.primitive || isScalarUnsigned(field)) { return field.type.valueTypeName @@ -1177,6 +1439,46 @@ internal class KotlinSerializerSourceWriter(private val struct: KotlinSourceStru private fun KotlinSourceTypeNode.isCollectionOrMap(): Boolean = typeId == "Types.LIST" || typeId == "Types.SET" || typeId == "Types.MAP" + private fun isDirectCopyValue(type: KotlinSourceTypeNode): Boolean = + type.typeArguments.isEmpty() && + type.componentType == null && + when (type.typeId) { + "Types.BOOL", + "Types.INT8", + "Types.UINT8", + "Types.INT16", + "Types.UINT16", + "Types.INT32", + "Types.VARINT32", + "Types.UINT32", + "Types.VAR_UINT32", + "Types.INT64", + "Types.VARINT64", + "Types.TAGGED_INT64", + "Types.UINT64", + "Types.VAR_UINT64", + "Types.TAGGED_UINT64", + "Types.FLOAT32", + "Types.FLOAT64", + "Types.STRING", + "Types.DATE", + "Types.TIMESTAMP", + "Types.DURATION", + "Types.DECIMAL", + "Types.FLOAT16", + "Types.BFLOAT16" -> true + else -> false + } + + private fun copiesWithContainerSerializer(type: KotlinSourceTypeNode): Boolean = + when (type.collectionFactory) { + CollectionFactory.NONE, + CollectionFactory.MUTABLE_LIST, + CollectionFactory.MUTABLE_SET, + CollectionFactory.MUTABLE_MAP -> false + else -> true + } + private fun copyContainerExpression( type: KotlinSourceTypeNode, expression: String, @@ -1186,30 +1488,51 @@ internal class KotlinSerializerSourceWriter(private val struct: KotlinSourceStru val source = "copySource$depth" return "$expression?.let { $source -> ${copyContainerExpression(type.copy(nullable = false), source, depth + 1)} }" } + if (copiesWithContainerSerializer(type)) { + return "(copyContext.copyObject($expression) as ${type.valueTypeName.removeSuffix("?")})" + } val source = "copySource$depth" val target = "copyTarget$depth" - val valueType = type.valueTypeName.removeSuffix("?") val copied = when (type.typeId) { "Types.LIST" -> { val element = "copyElement$depth" val copiedElement = copyElementExpression(type.typeArguments[0], element, depth + 1) - "run { val $source = $expression; val $target = java.util.ArrayList($source.size); for ($element in $source) { $target.add($copiedElement) }; $target as $valueType }" + val targetBuild = copyTarget(type, source, target) + "run { val $source = $expression; val $target = ${targetBuild.init}; copyContext.reference($source, $target); for ($element in $source) { $target.add($copiedElement) }; ${targetBuild.result} }" } "Types.SET" -> { val element = "copyElement$depth" val copiedElement = copyElementExpression(type.typeArguments[0], element, depth + 1) - "run { val $source = $expression; val $target = java.util.LinkedHashSet($source.size); for ($element in $source) { $target.add($copiedElement) }; $target as $valueType }" + val targetBuild = copyTarget(type, source, target) + "run { val $source = $expression; val $target = ${targetBuild.init}; copyContext.reference($source, $target); for ($element in $source) { $target.add($copiedElement) }; ${targetBuild.result} }" } "Types.MAP" -> { val entry = "copyEntry$depth" val copiedKey = copyElementExpression(type.typeArguments[0], "$entry.key", depth + 1) val copiedValue = copyElementExpression(type.typeArguments[1], "$entry.value", depth + 2) - "run { val $source = $expression; val $target = java.util.LinkedHashMap($source.size); for ($entry in $source.entries) { $target[$copiedKey] = $copiedValue }; $target as $valueType }" + val targetBuild = copyTarget(type, source, target) + "run { val $source = $expression; val $target = ${targetBuild.init}; copyContext.reference($source, $target); for ($entry in $source.entries) { $target[$copiedKey] = $copiedValue }; ${targetBuild.result} }" } - else -> "copyContext.copyObject($expression) as $valueType" + else -> "copyContext.copyObject($expression) as ${type.valueTypeName.removeSuffix("?")}" } - return applyCollectionFactory(type, copied) + return copied + } + + private fun copyTarget( + type: KotlinSourceTypeNode, + source: String, + target: String + ): ContainerTarget { + val valueType = type.valueTypeName.removeSuffix("?") + fun castTarget(): String = "$target as $valueType" + return when (type.typeId) { + "Types.LIST" -> ContainerTarget("java.util.ArrayList($source.size)", castTarget()) + "Types.SET" -> ContainerTarget("java.util.LinkedHashSet($source.size)", castTarget()) + "Types.MAP" -> + ContainerTarget("java.util.LinkedHashMap($source.size)", castTarget()) + else -> ContainerTarget("null", castTarget()) + } } private fun copyElementExpression( @@ -1225,7 +1548,7 @@ internal class KotlinSerializerSourceWriter(private val struct: KotlinSourceStru if (arrayCopy != null) { return arrayCopy } - if (type.primitive || type.unsigned || type.typeId == "Types.STRING") { + if (isDirectCopyValue(type)) { return expression } if (type.isCollectionOrMap()) { diff --git a/kotlin/fory-kotlin-ksp/src/test/kotlin/org/apache/fory/kotlin/ksp/ProcessorValidationTest.kt b/kotlin/fory-kotlin-ksp/src/test/kotlin/org/apache/fory/kotlin/ksp/ProcessorValidationTest.kt index 6c62579090..a0253036b7 100644 --- a/kotlin/fory-kotlin-ksp/src/test/kotlin/org/apache/fory/kotlin/ksp/ProcessorValidationTest.kt +++ b/kotlin/fory-kotlin-ksp/src/test/kotlin/org/apache/fory/kotlin/ksp/ProcessorValidationTest.kt @@ -146,6 +146,161 @@ class ProcessorValidationTest { assertTrue(!source.contains("return User(name = field0!!)")) assertTrue(source.contains("constructorFieldIds = if (objectCreator.hasConstructorFields())")) assertTrue(source.contains("objectCreator.newInstanceWithArguments(*constructorArgs")) + assertTrue(source.contains("fieldValues[0] = value.name")) + assertFalse(source.contains("copyConstructorFieldValue(copyContext, value, value.name")) + assertFalse(source.contains("NATURAL_ORDER_COMPARATOR")) + assertFalse(source.contains("requireXlangNaturalOrdering")) + } + + @Test + fun copyUsesDirectScalarValues() { + fun scalar(name: String, typeName: String, typeId: String) = + KotlinSourceTypeNode( + rawClassExpression = "$typeName::class.java", + kotlinTypeName = typeName, + valueTypeName = name, + typeName = typeName, + typeId = typeId, + nullable = false, + trackingRef = false, + primitive = false, + unsigned = false, + ) + + val fields = + listOf( + KotlinSourceField( + id = 0, + name = "date", + type = scalar("java.time.LocalDate", "java.time.LocalDate", "Types.DATE"), + hasForyField = true, + foryFieldId = 1, + trackingRef = false, + dynamic = "AUTO", + arrayType = false, + hasDefault = false, + nullable = false, + propertyTypeName = "java.time.LocalDate", + ), + KotlinSourceField( + id = 1, + name = "instant", + type = scalar("java.time.Instant", "java.time.Instant", "Types.TIMESTAMP"), + hasForyField = true, + foryFieldId = 2, + trackingRef = false, + dynamic = "AUTO", + arrayType = false, + hasDefault = false, + nullable = false, + propertyTypeName = "java.time.Instant", + ), + KotlinSourceField( + id = 2, + name = "duration", + type = scalar("kotlin.time.Duration", "java.time.Duration", "Types.DURATION"), + hasForyField = true, + foryFieldId = 3, + trackingRef = false, + dynamic = "AUTO", + arrayType = false, + hasDefault = false, + nullable = false, + propertyTypeName = "kotlin.time.Duration", + ), + KotlinSourceField( + id = 3, + name = "decimal", + type = scalar("java.math.BigDecimal", "java.math.BigDecimal", "Types.DECIMAL"), + hasForyField = true, + foryFieldId = 4, + trackingRef = false, + dynamic = "AUTO", + arrayType = false, + hasDefault = false, + nullable = false, + propertyTypeName = "java.math.BigDecimal", + ), + KotlinSourceField( + id = 4, + name = "float16", + type = + scalar("org.apache.fory.type.Float16", "org.apache.fory.type.Float16", "Types.FLOAT16"), + hasForyField = true, + foryFieldId = 5, + trackingRef = false, + dynamic = "AUTO", + arrayType = false, + hasDefault = false, + nullable = false, + propertyTypeName = "org.apache.fory.type.Float16", + ), + KotlinSourceField( + id = 5, + name = "bfloat16", + type = + scalar( + "org.apache.fory.type.BFloat16", + "org.apache.fory.type.BFloat16", + "Types.BFLOAT16", + ), + hasForyField = true, + foryFieldId = 6, + trackingRef = false, + dynamic = "AUTO", + arrayType = false, + hasDefault = false, + nullable = false, + propertyTypeName = "org.apache.fory.type.BFloat16", + ), + KotlinSourceField( + id = 6, + name = "child", + type = + KotlinSourceTypeNode( + rawClassExpression = "example.Child::class.java", + kotlinTypeName = "example.Child", + valueTypeName = "Child", + typeName = "example.Child", + typeId = null, + nullable = false, + trackingRef = false, + primitive = false, + unsigned = false, + ), + hasForyField = true, + foryFieldId = 7, + trackingRef = false, + dynamic = "AUTO", + arrayType = false, + hasDefault = false, + nullable = false, + propertyTypeName = "Child", + ), + ) + val source = + KotlinSerializerSourceWriter( + KotlinSourceStruct( + packageName = "example", + typeName = "Scalars", + qualifiedTypeName = "example.Scalars", + serializerName = "Scalars_ForySerializer", + serializerVisibility = KotlinSerializerVisibility.PUBLIC, + fields = fields, + originatingFiles = emptyList(), + ) + ) + .write() + + for (field in fields.take(6)) { + assertTrue(source.contains("fieldValues[${field.id}] = value.${field.name}")) + assertFalse( + source.contains("copyConstructorFieldValue(copyContext, value, value.${field.name}") + ) + } + assertTrue( + source.contains("fieldValues[6] = copyConstructorFieldValue(copyContext, value, value.child") + ) } @Test @@ -241,6 +396,20 @@ class ProcessorValidationTest { collectionFactory = CollectionFactory.TREE_MAP, typeArguments = listOf(stringType, intType), ) + val nestedMapType = + KotlinSourceTypeNode( + rawClassExpression = "java.util.Map::class.java", + kotlinTypeName = "java.util.TreeMap>", + valueTypeName = "java.util.TreeMap>", + typeName = "java.util.Map", + typeId = "Types.MAP", + nullable = false, + trackingRef = false, + primitive = false, + unsigned = false, + collectionFactory = CollectionFactory.TREE_MAP, + typeArguments = listOf(stringType, setType), + ) val source = KotlinSerializerSourceWriter( KotlinSourceStruct( @@ -292,6 +461,20 @@ class ProcessorValidationTest { nullable = false, propertyTypeName = "java.util.List", constructorParameterName = "arrays", + ), + KotlinSourceField( + id = 3, + name = "nestedCounts", + type = nestedMapType, + hasForyField = true, + foryFieldId = 4, + trackingRef = false, + dynamic = "AUTO", + arrayType = false, + hasDefault = false, + nullable = false, + propertyTypeName = "java.util.TreeMap>", + constructorParameterName = "nestedCounts", ) ), originatingFiles = emptyList(), @@ -329,17 +512,40 @@ class ProcessorValidationTest { ) assertTrue( source.contains( - "fieldValues[0] = KotlinCollectionAdapters.toTreeMap(run { val copySource0 = value.counts; val copyTarget0 = java.util.LinkedHashMap(copySource0.size);" + "nestedCounts = run { val readSource0 = (field3!! as Map<*, *>); val readTarget0 = java.util.TreeMap();" + ) + ) + assertTrue( + source.contains("KotlinCollectionAdapters.toTreeSet((readEntry0.value as Collection<*>))") + ) + val comparatorGuardIndex = source.indexOf("requireXlangNaturalOrdering(\"example.User.counts\"") + assertTrue(comparatorGuardIndex >= 0) + assertTrue(comparatorGuardIndex < source.indexOf("val buffer = writeContext.buffer")) + assertTrue(source.contains("requireXlangNaturalOrdering(\"example.User.names element\"")) + assertTrue(source.contains("requireXlangNaturalOrdering(\"example.User.nestedCounts value\"")) + assertTrue(source.contains("private val NATURAL_ORDER_COMPARATOR")) + assertTrue(source.contains("if (guard1List0 is java.util.RandomAccess)")) + assertTrue(source.contains("for (guard1Element0 in guard1List0)")) + assertTrue( + source.contains( + "fieldValues[0] = (copyContext.copyObject(value.counts) as kotlin.collections.Map)" + ) + ) + assertTrue( + source.contains( + "fieldValues[1] = run { val copySource0 = value.names; val copyTarget0 = java.util.ArrayList(copySource0.size); copyContext.reference(copySource0, copyTarget0); for (copyElement0 in copySource0) { copyTarget0.add((copyContext.copyObject(copyElement0) as kotlin.collections.Set))" ) ) assertTrue( source.contains( - "fieldValues[1] = run { val copySource0 = value.names; val copyTarget0 = java.util.ArrayList(copySource0.size); for (copyElement0 in copySource0) { copyTarget0.add(KotlinCollectionAdapters.toTreeSet(run { val copySource2 = copyElement0;" + "fieldValues[3] = (copyContext.copyObject(value.nestedCounts) as java.util.TreeMap>)" ) ) assertTrue(source.contains("fieldValues[2] = run { val copySource0 = value.arrays;")) assertTrue(source.contains("copyTarget0.add(copyElement0.copyOf())")) assertFalse(source.contains("fieldValues[0] = copyConstructorFieldValue")) + assertFalse(source.contains("fieldValues[0] = KotlinCollectionAdapters.toTreeMap(run")) + assertFalse(source.contains("java.util.TreeMap((copySource0.comparator()")) } @Test diff --git a/kotlin/fory-kotlin-tests/src/main/kotlin/org/apache/fory/kotlin/xlang/KotlinXlangPeer.kt b/kotlin/fory-kotlin-tests/src/main/kotlin/org/apache/fory/kotlin/xlang/KotlinXlangPeer.kt index a9a62767af..470a159f73 100644 --- a/kotlin/fory-kotlin-tests/src/main/kotlin/org/apache/fory/kotlin/xlang/KotlinXlangPeer.kt +++ b/kotlin/fory-kotlin-tests/src/main/kotlin/org/apache/fory/kotlin/xlang/KotlinXlangPeer.kt @@ -21,6 +21,9 @@ package org.apache.fory.kotlin.xlang +import java.math.BigDecimal +import java.time.Instant +import java.time.LocalDate import java.util.TreeMap import java.util.TreeSet import java.util.concurrent.ConcurrentHashMap @@ -38,6 +41,7 @@ import org.apache.fory.annotation.ForyUnion import org.apache.fory.annotation.ForyUnknownCase import org.apache.fory.annotation.Ref import org.apache.fory.exception.ForyException +import org.apache.fory.exception.SerializationException import org.apache.fory.kotlin.Fixed import org.apache.fory.kotlin.ForyKotlin import org.apache.fory.kotlin.VarInt @@ -45,7 +49,9 @@ import org.apache.fory.kotlin.register import org.apache.fory.memory.MemoryUtils import org.apache.fory.serializer.StaticGeneratedStructSerializer import org.apache.fory.serializer.kotlin.KotlinSerializers +import org.apache.fory.type.BFloat16 import org.apache.fory.type.BFloat16Array +import org.apache.fory.type.Float16 import org.apache.fory.type.Float16Array import org.apache.fory.type.Types import org.apache.fory.type.union.UnknownCase @@ -224,11 +230,25 @@ constructor( @ForyStruct public data class KotlinDurationAndHalfArrays -@ForyConstructor("duration", "float16s", "bfloat16s") +@ForyConstructor( + "duration", + "date", + "instant", + "decimal", + "float16", + "bfloat16", + "float16s", + "bfloat16s", +) constructor( @ForyField(id = 1) val duration: kotlin.time.Duration, - @ForyField(id = 2) val float16s: Float16Array, - @ForyField(id = 3) val bfloat16s: BFloat16Array, + @ForyField(id = 2) val date: LocalDate, + @ForyField(id = 3) val instant: Instant, + @ForyField(id = 4) val decimal: BigDecimal, + @ForyField(id = 5) val float16: Float16, + @ForyField(id = 6) val bfloat16: BFloat16, + @ForyField(id = 7) val float16s: Float16Array, + @ForyField(id = 8) val bfloat16s: BFloat16Array, ) @ForyStruct @@ -424,6 +444,23 @@ private fun staticSerializerRoundTrip(dataFile: String) { decoded.mutableCounts["round-trip-mutable-map"] = 14L check(decoded.sortedNames.javaClass == TreeSet::class.java) check(decoded.concurrentCounts.javaClass == ConcurrentHashMap::class.java) + val reverseCounts = TreeMap(java.util.Collections.reverseOrder()) + reverseCounts.putAll(mapOf("x" to 7, "y" to 8)) + val reverseSortedNames = TreeSet(java.util.Collections.reverseOrder()) + reverseSortedNames.addAll(listOf("e", "f")) + val copyCollections = collections.copy(counts = reverseCounts, sortedNames = reverseSortedNames) + try { + fory.serialize(copyCollections) + error("Kotlin concrete sorted collections with custom comparators were serialized") + } catch (e: SerializationException) { + check(e.cause is UnsupportedOperationException) + } + val copiedCollections = fory.copy(copyCollections) + check(copiedCollections == copyCollections) + check(copiedCollections.counts !== copyCollections.counts) + check(checkNotNull(copiedCollections.counts.comparator()).compare("a", "b") > 0) + check(copiedCollections.sortedNames !== copyCollections.sortedNames) + check(checkNotNull(copiedCollections.sortedNames.comparator()).compare("a", "b") > 0) val unsignedCollections = KotlinUnsignedCollections( @@ -504,6 +541,11 @@ private fun staticSerializerRoundTrip(dataFile: String) { val durationAndHalfArrays = KotlinDurationAndHalfArrays( duration = (-500).milliseconds, + date = LocalDate.of(2026, 6, 2), + instant = Instant.parse("2026-06-02T04:00:00Z"), + decimal = BigDecimal("12345.6789"), + float16 = Float16.valueOf(1.5f), + bfloat16 = BFloat16.valueOf(-2.5f), float16s = Float16Array.of(1.0f, -2.0f), bfloat16s = BFloat16Array.of(3.0f, -4.0f), ) @@ -513,8 +555,19 @@ private fun staticSerializerRoundTrip(dataFile: String) { KotlinDurationAndHalfArrays::class.java, ) check(decodedDurationAndHalfArrays.duration == durationAndHalfArrays.duration) + check(decodedDurationAndHalfArrays.date == durationAndHalfArrays.date) + check(decodedDurationAndHalfArrays.instant == durationAndHalfArrays.instant) + check(decodedDurationAndHalfArrays.decimal == durationAndHalfArrays.decimal) + check(decodedDurationAndHalfArrays.float16 == durationAndHalfArrays.float16) + check(decodedDurationAndHalfArrays.bfloat16 == durationAndHalfArrays.bfloat16) check(decodedDurationAndHalfArrays.float16s == durationAndHalfArrays.float16s) check(decodedDurationAndHalfArrays.bfloat16s == durationAndHalfArrays.bfloat16s) + val copiedDurationAndHalfArrays = fory.copy(durationAndHalfArrays) + check(copiedDurationAndHalfArrays.date === durationAndHalfArrays.date) + check(copiedDurationAndHalfArrays.instant === durationAndHalfArrays.instant) + check(copiedDurationAndHalfArrays.decimal === durationAndHalfArrays.decimal) + check(copiedDurationAndHalfArrays.float16 === durationAndHalfArrays.float16) + check(copiedDurationAndHalfArrays.bfloat16 === durationAndHalfArrays.bfloat16) try { fory.serialize(durationAndHalfArrays.copy(duration = Duration.INFINITE)) error("Infinite Kotlin xlang duration was serialized") From 0e3e9c566dfa180db5a97a5968504b2f3bf95e28 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Tue, 2 Jun 2026 14:36:51 +0800 Subject: [PATCH 42/69] refactor(java): clean jdk25 overlay branches --- .../org/apache/fory/memory/LittleEndian.java | 18 +- .../org/apache/fory/memory/MemoryBuffer.java | 1029 +++++++---------- .../fory/platform/internal/DefineClass.java | 4 +- .../serializer/SerializationHookLookup.java | 3 +- 4 files changed, 429 insertions(+), 625 deletions(-) diff --git a/java/fory-core/src/main/java25/org/apache/fory/memory/LittleEndian.java b/java/fory-core/src/main/java25/org/apache/fory/memory/LittleEndian.java index bc7ffd4e9d..3687b8ec59 100644 --- a/java/fory-core/src/main/java25/org/apache/fory/memory/LittleEndian.java +++ b/java/fory-core/src/main/java25/org/apache/fory/memory/LittleEndian.java @@ -56,10 +56,24 @@ private static int bigWriteUint36(byte[] arr, int index, long v) { } public static long getInt64(byte[] o, int index) { - return MemoryOps.getInt64(o, index); + return ((long) o[index] & 0xff) + | (((long) o[index + 1] & 0xff) << 8) + | (((long) o[index + 2] & 0xff) << 16) + | (((long) o[index + 3] & 0xff) << 24) + | (((long) o[index + 4] & 0xff) << 32) + | (((long) o[index + 5] & 0xff) << 40) + | (((long) o[index + 6] & 0xff) << 48) + | (((long) o[index + 7] & 0xff) << 56); } public static void putInt64(byte[] o, int index, long value) { - MemoryOps.putInt64(o, index, value); + o[index] = (byte) value; + o[index + 1] = (byte) (value >>> 8); + o[index + 2] = (byte) (value >>> 16); + o[index + 3] = (byte) (value >>> 24); + o[index + 4] = (byte) (value >>> 32); + o[index + 5] = (byte) (value >>> 40); + o[index + 6] = (byte) (value >>> 48); + o[index + 7] = (byte) (value >>> 56); } } diff --git a/java/fory-core/src/main/java25/org/apache/fory/memory/MemoryBuffer.java b/java/fory-core/src/main/java25/org/apache/fory/memory/MemoryBuffer.java index 8472bfd3f0..66308eb987 100644 --- a/java/fory-core/src/main/java25/org/apache/fory/memory/MemoryBuffer.java +++ b/java/fory-core/src/main/java25/org/apache/fory/memory/MemoryBuffer.java @@ -197,7 +197,8 @@ private MemoryBuffer( private void initOffHeapBuffer(long offHeapAddress, int size, ByteBuffer offHeapBuffer) { checkArgument(offHeapAddress >= 0 && size >= 0); checkNotNull(offHeapBuffer, "JDK25 MemoryBuffer requires a ByteBuffer owner for off-heap data"); - checkArgument(offHeapBuffer.isDirect(), "Only direct ByteBuffers can back off-heap MemoryBuffer"); + checkArgument( + offHeapBuffer.isDirect(), "Only direct ByteBuffers can back off-heap MemoryBuffer"); this.offHeapBuffer = offHeapBuffer; ByteBuffer nativeBuffer = offHeapBuffer.duplicate().order(NATIVE_ORDER); // Stream readers can expand the owner buffer limit after this duplicate is created. Keep the @@ -219,9 +220,7 @@ private static long getAddress(ByteBuffer buffer) { public void initByteBuffer(ByteBuffer buffer, int size) { if (buffer.isDirect()) { - - initOffHeapBuffer(0, size, buffer); - + initOffHeapBuffer(0, size, buffer); } else if (buffer.hasArray()) { initHeapBuffer(buffer.array(), buffer.arrayOffset(), size); } else { @@ -261,19 +260,17 @@ public MemoryBuffer getBuffer() { } public void initHeapBuffer(byte[] buffer, int offset, int length) { - - if (buffer == null) { - throw new NullPointerException("buffer"); - } - this.heapMemory = buffer; - this.heapOffset = offset; - this.offHeapBuffer = null; - this.nativeOffHeapBuffer = null; - final long startPos = BYTE_ARRAY_OFFSET + offset; - this.address = startPos; - this.size = length; - this.addressLimit = startPos + length; - + if (buffer == null) { + throw new NullPointerException("buffer"); + } + this.heapMemory = buffer; + this.heapOffset = offset; + this.offHeapBuffer = null; + this.nativeOffHeapBuffer = null; + final long startPos = BYTE_ARRAY_OFFSET + offset; + this.address = startPos; + this.size = length; + this.addressLimit = startPos + length; } private byte loadByte(long pos) { @@ -370,7 +367,8 @@ private void readBytesToArray(long srcOffset, byte[] target, int targetOffset, i } } - private void writeBytesFromArray(long targetOffset, byte[] source, int sourceOffset, int numBytes) { + private void writeBytesFromArray( + long targetOffset, byte[] source, int sourceOffset, int numBytes) { byte[] heap = heapMemory; if (heap != null) { System.arraycopy(source, sourceOffset, heap, toIntIndex(targetOffset), numBytes); @@ -409,7 +407,8 @@ private void readCharsToArray(long srcOffset, char[] target, int targetOffset, i } } - private void writeCharsFromArray(long targetOffset, char[] source, int sourceOffset, int numBytes) { + private void writeCharsFromArray( + long targetOffset, char[] source, int sourceOffset, int numBytes) { int elements = numBytes >>> 1; int pos = toIntIndex(targetOffset); byte[] heap = heapMemory; @@ -506,7 +505,8 @@ private void readLongsToArray(long srcOffset, long[] target, int targetOffset, i } } - private void writeLongsFromArray(long targetOffset, long[] source, int sourceOffset, int numBytes) { + private void writeLongsFromArray( + long targetOffset, long[] source, int sourceOffset, int numBytes) { int elements = numBytes >>> 3; int pos = toIntIndex(targetOffset); byte[] heap = heapMemory; @@ -566,7 +566,8 @@ private void readDoublesToArray(long srcOffset, double[] target, int targetOffse } else { ByteBuffer direct = nativeOffHeapBuffer; for (int i = 0; i < elements; i++, pos += 8) { - target[targetOffset + i] = Double.longBitsToDouble((long) BYTE_BUFFER_LONG.get(direct, pos)); + target[targetOffset + i] = + Double.longBitsToDouble((long) BYTE_BUFFER_LONG.get(direct, pos)); } } } @@ -812,7 +813,6 @@ public void put(int index, byte[] src, int offset, int length) { } public byte getByte(int index) { - final long pos = address + index; checkPosition(index, pos, 1); return loadByte(pos); @@ -821,36 +821,28 @@ public byte getByte(int index) { // CHECKSTYLE.OFF:MethodName public byte _unsafeGetByte(int index) { // CHECKSTYLE.ON:MethodName - return loadByte(address + index); } public void putByte(int index, int b) { - - final long pos = address + index; - checkPosition(index, pos, 1); - storeByte(pos, (byte) b); - + final long pos = address + index; + checkPosition(index, pos, 1); + storeByte(pos, (byte) b); } public void putByte(int index, byte b) { - - final long pos = address + index; - checkPosition(index, pos, 1); - storeByte(pos, b); - + final long pos = address + index; + checkPosition(index, pos, 1); + storeByte(pos, b); } // CHECKSTYLE.OFF:MethodName public void _unsafePutByte(int index, byte b) { // CHECKSTYLE.ON:MethodName - - storeByte(address + index, b); - + storeByte(address + index, b); } public boolean getBoolean(int index) { - final long pos = address + index; checkPosition(index, pos, 1); return loadByte(pos) != 0; @@ -859,26 +851,20 @@ public boolean getBoolean(int index) { // CHECKSTYLE.OFF:MethodName public boolean _unsafeGetBoolean(int index) { // CHECKSTYLE.ON:MethodName - return loadByte(address + index) != 0; } public void putBoolean(int index, boolean value) { - - storeByte(address + index, (value ? (byte) 1 : (byte) 0)); - + storeByte(address + index, (value ? (byte) 1 : (byte) 0)); } // CHECKSTYLE.OFF:MethodName public void _unsafePutBoolean(int index, boolean value) { // CHECKSTYLE.ON:MethodName - - storeByte(address + index, (value ? (byte) 1 : (byte) 0)); - + storeByte(address + index, (value ? (byte) 1 : (byte) 0)); } public char getChar(int index) { - final long pos = address + index; checkPosition(index, pos, 2); char c = loadChar(pos); @@ -888,35 +874,29 @@ public char getChar(int index) { // CHECKSTYLE.OFF:MethodName public char _unsafeGetChar(int index) { // CHECKSTYLE.ON:MethodName - char c = loadChar(address + index); return LITTLE_ENDIAN ? c : Character.reverseBytes(c); } public void putChar(int index, char value) { - - final long pos = address + index; - checkPosition(index, pos, 2); - if (!LITTLE_ENDIAN) { - value = Character.reverseBytes(value); - } - storeChar(pos, value); - + final long pos = address + index; + checkPosition(index, pos, 2); + if (!LITTLE_ENDIAN) { + value = Character.reverseBytes(value); + } + storeChar(pos, value); } // CHECKSTYLE.OFF:MethodName public void _unsafePutChar(int index, char value) { // CHECKSTYLE.ON:MethodName - - if (!LITTLE_ENDIAN) { - value = Character.reverseBytes(value); - } - storeChar(address + index, value); - + if (!LITTLE_ENDIAN) { + value = Character.reverseBytes(value); + } + storeChar(address + index, value); } public short getInt16(int index) { - final long pos = address + index; checkPosition(index, pos, 2); short v = loadShort(pos); @@ -924,20 +904,17 @@ public short getInt16(int index) { } public void putInt16(int index, short value) { - - final long pos = address + index; - checkPosition(index, pos, 2); - if (!LITTLE_ENDIAN) { - value = Short.reverseBytes(value); - } - storeShort(pos, value); - + final long pos = address + index; + checkPosition(index, pos, 2); + if (!LITTLE_ENDIAN) { + value = Short.reverseBytes(value); + } + storeShort(pos, value); } // CHECKSTYLE.OFF:MethodName public short _unsafeGetInt16(int index) { // CHECKSTYLE.ON:MethodName - short v = loadShort(address + index); return LITTLE_ENDIAN ? v : Short.reverseBytes(v); } @@ -945,16 +922,13 @@ public short _unsafeGetInt16(int index) { // CHECKSTYLE.OFF:MethodName public void _unsafePutInt16(int index, short value) { // CHECKSTYLE.ON:MethodName - - if (!LITTLE_ENDIAN) { - value = Short.reverseBytes(value); - } - storeShort(address + index, value); - + if (!LITTLE_ENDIAN) { + value = Short.reverseBytes(value); + } + storeShort(address + index, value); } public int getInt32(int index) { - final long pos = address + index; checkPosition(index, pos, 4); int v = loadInt(pos); @@ -962,20 +936,17 @@ public int getInt32(int index) { } public void putInt32(int index, int value) { - - final long pos = address + index; - checkPosition(index, pos, 4); - if (!LITTLE_ENDIAN) { - value = Integer.reverseBytes(value); - } - storeInt(pos, value); - + final long pos = address + index; + checkPosition(index, pos, 4); + if (!LITTLE_ENDIAN) { + value = Integer.reverseBytes(value); + } + storeInt(pos, value); } // CHECKSTYLE.OFF:MethodName public int _unsafeGetInt32(int index) { // CHECKSTYLE.ON:MethodName - int v = loadInt(address + index); return LITTLE_ENDIAN ? v : Integer.reverseBytes(v); } @@ -983,16 +954,13 @@ public int _unsafeGetInt32(int index) { // CHECKSTYLE.OFF:MethodName public void _unsafePutInt32(int index, int value) { // CHECKSTYLE.ON:MethodName - - if (!LITTLE_ENDIAN) { - value = Integer.reverseBytes(value); - } - storeInt(address + index, value); - + if (!LITTLE_ENDIAN) { + value = Integer.reverseBytes(value); + } + storeInt(address + index, value); } public long getInt64(int index) { - final long pos = address + index; checkPosition(index, pos, 8); long v = loadLong(pos); @@ -1000,20 +968,17 @@ public long getInt64(int index) { } public void putInt64(int index, long value) { - - final long pos = address + index; - checkPosition(index, pos, 8); - if (!LITTLE_ENDIAN) { - value = Long.reverseBytes(value); - } - storeLong(pos, value); - + final long pos = address + index; + checkPosition(index, pos, 8); + if (!LITTLE_ENDIAN) { + value = Long.reverseBytes(value); + } + storeLong(pos, value); } // CHECKSTYLE.OFF:MethodName public long _unsafeGetInt64(int index) { // CHECKSTYLE.ON:MethodName - long v = loadLong(address + index); return LITTLE_ENDIAN ? v : Long.reverseBytes(v); } @@ -1021,16 +986,13 @@ public long _unsafeGetInt64(int index) { // CHECKSTYLE.OFF:MethodName public void _unsafePutInt64(int index, long value) { // CHECKSTYLE.ON:MethodName - - if (!LITTLE_ENDIAN) { - value = Long.reverseBytes(value); - } - storeLong(address + index, value); - + if (!LITTLE_ENDIAN) { + value = Long.reverseBytes(value); + } + storeLong(address + index, value); } public float getFloat32(int index) { - final long pos = address + index; checkPosition(index, pos, 4); int v = loadInt(pos); @@ -1041,19 +1003,16 @@ public float getFloat32(int index) { } public void putFloat32(int index, float value) { - - final long pos = address + index; - checkPosition(index, pos, 4); - int v = Float.floatToRawIntBits(value); - if (!LITTLE_ENDIAN) { - v = Integer.reverseBytes(v); - } - storeInt(pos, v); - + final long pos = address + index; + checkPosition(index, pos, 4); + int v = Float.floatToRawIntBits(value); + if (!LITTLE_ENDIAN) { + v = Integer.reverseBytes(v); + } + storeInt(pos, v); } public double getFloat64(int index) { - final long pos = address + index; checkPosition(index, pos, 8); long v = loadLong(pos); @@ -1064,15 +1023,13 @@ public double getFloat64(int index) { } public void putFloat64(int index, double value) { - - final long pos = address + index; - checkPosition(index, pos, 8); - long v = Double.doubleToRawLongBits(value); - if (!LITTLE_ENDIAN) { - v = Long.reverseBytes(v); - } - storeLong(pos, v); - + final long pos = address + index; + checkPosition(index, pos, 8); + long v = Double.doubleToRawLongBits(value); + if (!LITTLE_ENDIAN) { + v = Long.reverseBytes(v); + } + storeLong(pos, v); } // Check should be done outside to avoid this method got into the critical path. @@ -1143,26 +1100,22 @@ public void increaseWriterIndex(int diff) { } public void writeBoolean(boolean value) { - - final int writerIdx = writerIndex; - final int newIdx = writerIdx + 1; - ensure(newIdx); - final long pos = address + writerIdx; - storeByte(pos, (byte) (value ? 1 : 0)); - writerIndex = newIdx; - + final int writerIdx = writerIndex; + final int newIdx = writerIdx + 1; + ensure(newIdx); + final long pos = address + writerIdx; + storeByte(pos, (byte) (value ? 1 : 0)); + writerIndex = newIdx; } // CHECKSTYLE.OFF:MethodName public void _unsafeWriteByte(byte value) { // CHECKSTYLE.ON:MethodName - - final int writerIdx = writerIndex; - final int newIdx = writerIdx + 1; - final long pos = address + writerIdx; - storeByte(pos, value); - writerIndex = newIdx; - + final int writerIdx = writerIndex; + final int newIdx = writerIdx + 1; + final long pos = address + writerIdx; + storeByte(pos, value); + writerIndex = newIdx; } public void writeUInt8(int value) { @@ -1170,14 +1123,12 @@ public void writeUInt8(int value) { } public void writeByte(byte value) { - - final int writerIdx = writerIndex; - final int newIdx = writerIdx + 1; - ensure(newIdx); - final long pos = address + writerIdx; - storeByte(pos, value); - writerIndex = newIdx; - + final int writerIdx = writerIndex; + final int newIdx = writerIdx + 1; + ensure(newIdx); + final long pos = address + writerIdx; + storeByte(pos, value); + writerIndex = newIdx; } public void writeByte(int value) { @@ -1185,84 +1136,72 @@ public void writeByte(int value) { } public void writeChar(char value) { - - final int writerIdx = writerIndex; - final int newIdx = writerIdx + 2; - ensure(newIdx); - final long pos = address + writerIdx; - if (!LITTLE_ENDIAN) { - value = Character.reverseBytes(value); - } - storeChar(pos, value); - writerIndex = newIdx; - + final int writerIdx = writerIndex; + final int newIdx = writerIdx + 2; + ensure(newIdx); + final long pos = address + writerIdx; + if (!LITTLE_ENDIAN) { + value = Character.reverseBytes(value); + } + storeChar(pos, value); + writerIndex = newIdx; } public void writeInt16(short value) { - - final int writerIdx = writerIndex; - final int newIdx = writerIdx + 2; - ensure(newIdx); - if (!LITTLE_ENDIAN) { - value = Short.reverseBytes(value); - } - storeShort(address + writerIdx, value); - writerIndex = newIdx; - + final int writerIdx = writerIndex; + final int newIdx = writerIdx + 2; + ensure(newIdx); + if (!LITTLE_ENDIAN) { + value = Short.reverseBytes(value); + } + storeShort(address + writerIdx, value); + writerIndex = newIdx; } public void writeInt32(int value) { - - final int writerIdx = writerIndex; - final int newIdx = writerIdx + 4; - ensure(newIdx); - if (!LITTLE_ENDIAN) { - value = Integer.reverseBytes(value); - } - storeInt(address + writerIdx, value); - writerIndex = newIdx; - + final int writerIdx = writerIndex; + final int newIdx = writerIdx + 4; + ensure(newIdx); + if (!LITTLE_ENDIAN) { + value = Integer.reverseBytes(value); + } + storeInt(address + writerIdx, value); + writerIndex = newIdx; } public void writeInt64(long value) { - - final int writerIdx = writerIndex; - final int newIdx = writerIdx + 8; - ensure(newIdx); - if (!LITTLE_ENDIAN) { - value = Long.reverseBytes(value); - } - storeLong(address + writerIdx, value); - writerIndex = newIdx; - + final int writerIdx = writerIndex; + final int newIdx = writerIdx + 8; + ensure(newIdx); + if (!LITTLE_ENDIAN) { + value = Long.reverseBytes(value); + } + storeLong(address + writerIdx, value); + writerIndex = newIdx; } public void writeFloat32(float value) { - - final int writerIdx = writerIndex; - final int newIdx = writerIdx + 4; - ensure(newIdx); - int v = Float.floatToRawIntBits(value); - if (!LITTLE_ENDIAN) { - v = Integer.reverseBytes(v); - } - storeInt(address + writerIdx, v); - writerIndex = newIdx; - + final int writerIdx = writerIndex; + final int newIdx = writerIdx + 4; + ensure(newIdx); + int v = Float.floatToRawIntBits(value); + if (!LITTLE_ENDIAN) { + v = Integer.reverseBytes(v); + } + storeInt(address + writerIdx, v); + writerIndex = newIdx; } public void writeFloat64(double value) { - - final int writerIdx = writerIndex; - final int newIdx = writerIdx + 8; - ensure(newIdx); - long v = Double.doubleToRawLongBits(value); - if (!LITTLE_ENDIAN) { - v = Long.reverseBytes(v); - } - storeLong(address + writerIdx, v); - writerIndex = newIdx; - + final int writerIdx = writerIndex; + final int newIdx = writerIdx + 8; + ensure(newIdx); + long v = Double.doubleToRawLongBits(value); + if (!LITTLE_ENDIAN) { + v = Long.reverseBytes(v); + } + storeLong(address + writerIdx, v); + writerIndex = newIdx; } /** @@ -1270,7 +1209,6 @@ public void writeFloat64(double value) { * to save one bit. */ public int writeVarInt32(int v) { - ensure(writerIndex + 8); // Zigzag encoding: maps negative values to positive values // This works entirely in int without conversion to long @@ -1287,7 +1225,6 @@ public int writeVarInt32(int v) { // CHECKSTYLE.OFF:MethodName public int _unsafeWriteVarInt32(int v) { // CHECKSTYLE.ON:MethodName - // Zigzag encoding ensures negatives close to zero are encoded in few bytes int varintBytes = _unsafePutVarUInt32(writerIndex, (v << 1) ^ (v >> 31)); writerIndex += varintBytes; @@ -1300,7 +1237,6 @@ public int _unsafeWriteVarInt32(int v) { * @return The number of bytes written. */ public int writeVarUInt32(int v) { - // ensure at least 8 bytes are writable at once, so jvm-jit // generated code is smaller. Otherwise, the reference writer fast path // may be `callee is too large`/`already compiled into a big method` @@ -1317,7 +1253,6 @@ public int writeVarUInt32(int v) { // CHECKSTYLE.OFF:MethodName public int _unsafeWriteVarUInt32(int v) { // CHECKSTYLE.ON:MethodName - int varintBytes = _unsafePutVarUInt32(writerIndex, v); writerIndex += varintBytes; return varintBytes; @@ -1328,7 +1263,6 @@ public int _unsafeWriteVarUInt32(int v) { * 127). When the value is equal or greater than 127, the write will be a little slower. */ public int writeVarUInt32Small7(int value) { - ensure(writerIndex + 8); if (value >>> 7 == 0) { storeByte(address + writerIndex++, (byte) value); @@ -1383,7 +1317,6 @@ private int continueWriteVarUInt32Small7(int value) { // CHECKSTYLE.OFF:MethodName public int _unsafePutVarUInt32(int index, int value) { // CHECKSTYLE.ON:MethodName - byte[] heap = heapMemory; if (heap != null) { return putVarUInt32Heap(heap, (int) (address + index), value); @@ -1503,7 +1436,6 @@ private int continuePutVarUInt32BigEndian(int index, int encoded, int value) { // CHECKSTYLE.OFF:MethodName public int _unsafePutVarUint36Small(int index, long value) { // CHECKSTYLE.ON:MethodName - long encoded = (value & 0x7F); if (value >>> 7 == 0) { storeByte(address + index, (byte) value); @@ -1574,7 +1506,6 @@ private int continuePutVarUint36SmallBigEndian(int index, long encoded, long val * @return The number of bytes written. */ public int writeVarUInt32Aligned(int value) { - // Mask first 6 bits, // bit 7 `unset` indicates have next padding bytes, // bit 8 `set` indicates have next data bytes. @@ -1767,7 +1698,6 @@ private int writeVarUInt32Aligned6(int value) { * #writeVarUInt64} to save one bit. */ public int writeVarInt64(long value) { - ensure(writerIndex + 9); return _unsafeWriteVarUInt64((value << 1) ^ (value >> 63)); } @@ -1776,12 +1706,10 @@ public int writeVarInt64(long value) { // CHECKSTYLE.OFF:MethodName public int _unsafeWriteVarInt64(long value) { // CHECKSTYLE.ON:MethodName - return _unsafeWriteVarUInt64((value << 1) ^ (value >> 63)); } public int writeVarUInt64(long value) { - // Var long encoding algorithm is based kryo UnsafeMemoryOutput.writeVarInt64. // var long are written using little endian byte order. ensure(writerIndex + 9); @@ -1792,7 +1720,6 @@ public int writeVarUInt64(long value) { @CodegenInvoke public int _unsafeWriteVarUInt64(long value) { // CHECKSTYLE.ON:MethodName - final int writerIndex = this.writerIndex; int varInt; varInt = (int) (value & 0x7F); @@ -1875,7 +1802,6 @@ public int writeTaggedUInt64(long value) { // CHECKSTYLE.OFF:MethodName public int _unsafeWriteTaggedUInt64(long value) { // CHECKSTYLE.ON:MethodName - final int writerIndex = this.writerIndex; final long pos = address + writerIndex; final byte[] heapMemory = this.heapMemory; @@ -1906,7 +1832,6 @@ public int _unsafeWriteTaggedUInt64(long value) { // CHECKSTYLE.OFF:MethodName public int _unsafeWriteTaggedInt64(long value) { // CHECKSTYLE.ON:MethodName - final int writerIndex = this.writerIndex; final long pos = address + writerIndex; final byte[] heapMemory = this.heapMemory; @@ -1960,17 +1885,13 @@ public void write(ByteBuffer source, int numBytes) { } public void writeBytesWithSize(byte[] values) { - - writeVarUInt32Small7(values.length); - writeBytes(values, 0, values.length); - + writeVarUInt32Small7(values.length); + writeBytes(values, 0, values.length); } public void writeBooleansWithSize(boolean[] values) { - - writeVarUInt32Small7(values.length); - writeBooleans(values, 0, values.length); - + writeVarUInt32Small7(values.length); + writeBooleans(values, 0, values.length); } public void writeBooleans(boolean[] values) { @@ -1978,21 +1899,17 @@ public void writeBooleans(boolean[] values) { } public void writeBooleans(boolean[] values, int offset, int numElements) { - - final int writerIdx = writerIndex; - final int newIdx = writerIdx + numElements; - ensure(newIdx); - writeBooleansFromArray(address + writerIdx, values, BOOLEAN_ARRAY_OFFSET + offset, numElements); - writerIndex = newIdx; - + final int writerIdx = writerIndex; + final int newIdx = writerIdx + numElements; + ensure(newIdx); + writeBooleansFromArray(address + writerIdx, values, BOOLEAN_ARRAY_OFFSET + offset, numElements); + writerIndex = newIdx; } public void writeCharsWithSize(char[] values) { - - int numBytes = Math.multiplyExact(values.length, 2); - writeVarUInt32Small7(numBytes); - writeChars(values, 0, values.length); - + int numBytes = Math.multiplyExact(values.length, 2); + writeVarUInt32Small7(numBytes); + writeChars(values, 0, values.length); } public void writeChars(char[] values) { @@ -2000,22 +1917,18 @@ public void writeChars(char[] values) { } public void writeChars(char[] values, int offset, int numElements) { - - int numBytes = Math.multiplyExact(numElements, 2); - final int writerIdx = writerIndex; - final int newIdx = writerIdx + numBytes; - ensure(newIdx); - writeCharsFromArray(address + writerIdx, values, CHAR_ARRAY_OFFSET + offset, numBytes); - writerIndex = newIdx; - + int numBytes = Math.multiplyExact(numElements, 2); + final int writerIdx = writerIndex; + final int newIdx = writerIdx + numBytes; + ensure(newIdx); + writeCharsFromArray(address + writerIdx, values, CHAR_ARRAY_OFFSET + offset, numBytes); + writerIndex = newIdx; } public void writeShortsWithSize(short[] values) { - - int numBytes = Math.multiplyExact(values.length, 2); - writeVarUInt32Small7(numBytes); - writeShorts(values, 0, values.length); - + int numBytes = Math.multiplyExact(values.length, 2); + writeVarUInt32Small7(numBytes); + writeShorts(values, 0, values.length); } public void writeShorts(short[] values) { @@ -2023,22 +1936,18 @@ public void writeShorts(short[] values) { } public void writeShorts(short[] values, int offset, int numElements) { - - int numBytes = Math.multiplyExact(numElements, 2); - final int writerIdx = writerIndex; - final int newIdx = writerIdx + numBytes; - ensure(newIdx); - writeShortsFromArray(address + writerIdx, values, SHORT_ARRAY_OFFSET + offset, numBytes); - writerIndex = newIdx; - + int numBytes = Math.multiplyExact(numElements, 2); + final int writerIdx = writerIndex; + final int newIdx = writerIdx + numBytes; + ensure(newIdx); + writeShortsFromArray(address + writerIdx, values, SHORT_ARRAY_OFFSET + offset, numBytes); + writerIndex = newIdx; } public void writeIntsWithSize(int[] values) { - - int numBytes = Math.multiplyExact(values.length, 4); - writeVarUInt32Small7(numBytes); - writeInts(values, 0, values.length); - + int numBytes = Math.multiplyExact(values.length, 4); + writeVarUInt32Small7(numBytes); + writeInts(values, 0, values.length); } public void writeInts(int[] values) { @@ -2046,22 +1955,18 @@ public void writeInts(int[] values) { } public void writeInts(int[] values, int offset, int numElements) { - - int numBytes = Math.multiplyExact(numElements, 4); - final int writerIdx = writerIndex; - final int newIdx = writerIdx + numBytes; - ensure(newIdx); - writeIntsFromArray(address + writerIdx, values, INT_ARRAY_OFFSET + offset, numBytes); - writerIndex = newIdx; - + int numBytes = Math.multiplyExact(numElements, 4); + final int writerIdx = writerIndex; + final int newIdx = writerIdx + numBytes; + ensure(newIdx); + writeIntsFromArray(address + writerIdx, values, INT_ARRAY_OFFSET + offset, numBytes); + writerIndex = newIdx; } public void writeLongsWithSize(long[] values) { - - int numBytes = Math.multiplyExact(values.length, 8); - writeVarUInt32Small7(numBytes); - writeLongs(values, 0, values.length); - + int numBytes = Math.multiplyExact(values.length, 8); + writeVarUInt32Small7(numBytes); + writeLongs(values, 0, values.length); } public void writeLongs(long[] values) { @@ -2069,22 +1974,18 @@ public void writeLongs(long[] values) { } public void writeLongs(long[] values, int offset, int numElements) { - - int numBytes = Math.multiplyExact(numElements, 8); - final int writerIdx = writerIndex; - final int newIdx = writerIdx + numBytes; - ensure(newIdx); - writeLongsFromArray(address + writerIdx, values, LONG_ARRAY_OFFSET + offset, numBytes); - writerIndex = newIdx; - + int numBytes = Math.multiplyExact(numElements, 8); + final int writerIdx = writerIndex; + final int newIdx = writerIdx + numBytes; + ensure(newIdx); + writeLongsFromArray(address + writerIdx, values, LONG_ARRAY_OFFSET + offset, numBytes); + writerIndex = newIdx; } public void writeFloatsWithSize(float[] values) { - - int numBytes = Math.multiplyExact(values.length, 4); - writeVarUInt32Small7(numBytes); - writeFloats(values, 0, values.length); - + int numBytes = Math.multiplyExact(values.length, 4); + writeVarUInt32Small7(numBytes); + writeFloats(values, 0, values.length); } public void writeFloats(float[] values) { @@ -2092,22 +1993,18 @@ public void writeFloats(float[] values) { } public void writeFloats(float[] values, int offset, int numElements) { - - int numBytes = Math.multiplyExact(numElements, 4); - final int writerIdx = writerIndex; - final int newIdx = writerIdx + numBytes; - ensure(newIdx); - writeFloatsFromArray(address + writerIdx, values, FLOAT_ARRAY_OFFSET + offset, numBytes); - writerIndex = newIdx; - + int numBytes = Math.multiplyExact(numElements, 4); + final int writerIdx = writerIndex; + final int newIdx = writerIdx + numBytes; + ensure(newIdx); + writeFloatsFromArray(address + writerIdx, values, FLOAT_ARRAY_OFFSET + offset, numBytes); + writerIndex = newIdx; } public void writeDoublesWithSize(double[] values) { - - int numBytes = Math.multiplyExact(values.length, 8); - writeVarUInt32Small7(numBytes); - writeDoubles(values, 0, values.length); - + int numBytes = Math.multiplyExact(values.length, 8); + writeVarUInt32Small7(numBytes); + writeDoubles(values, 0, values.length); } public void writeDoubles(double[] values) { @@ -2115,14 +2012,12 @@ public void writeDoubles(double[] values) { } public void writeDoubles(double[] values, int offset, int numElements) { - - int numBytes = Math.multiplyExact(numElements, 8); - final int writerIdx = writerIndex; - final int newIdx = writerIdx + numBytes; - ensure(newIdx); - writeDoublesFromArray(address + writerIdx, values, DOUBLE_ARRAY_OFFSET + offset, numBytes); - writerIndex = newIdx; - + int numBytes = Math.multiplyExact(numElements, 8); + final int writerIdx = writerIndex; + final int newIdx = writerIdx + numBytes; + ensure(newIdx); + writeDoublesFromArray(address + writerIdx, values, DOUBLE_ARRAY_OFFSET + offset, numBytes); + writerIndex = newIdx; } /** For off-heap buffer, this will make a heap buffer internally. */ @@ -2221,7 +2116,6 @@ public int remaining() { } public boolean readBoolean() { - int readerIdx = readerIndex; // use subtract to avoid overflow if (readerIdx > size - 1) { @@ -2232,7 +2126,6 @@ public boolean readBoolean() { } public int readUInt8() { - int readerIdx = readerIndex; if (readerIdx > size - 1) { streamReader.fillBuffer(1); @@ -2248,7 +2141,6 @@ public int readUnsignedByte() { } public byte readByte() { - int readerIdx = readerIndex; if (readerIdx > size - 1) { streamReader.fillBuffer(1); @@ -2258,7 +2150,6 @@ public byte readByte() { } public char readChar() { - int readerIdx = readerIndex; // use subtract to avoid overflow int remaining = size - readerIdx; @@ -2271,7 +2162,6 @@ public char readChar() { } public short readInt16() { - int readerIdx = readerIndex; // use subtract to avoid overflow int remaining = size - readerIdx; @@ -2288,7 +2178,6 @@ public short readInt16() { // CHECKSTYLE.OFF:MethodName public short _readInt16OnLE() { // CHECKSTYLE.ON:MethodName - int readerIdx = readerIndex; // use subtract to avoid overflow int remaining = size - readerIdx; @@ -2304,7 +2193,6 @@ public short _readInt16OnLE() { // CHECKSTYLE.OFF:MethodName public short _readInt16OnBE() { // CHECKSTYLE.ON:MethodName - int readerIdx = readerIndex; // use subtract to avoid overflow int remaining = size - readerIdx; @@ -2316,7 +2204,6 @@ public short _readInt16OnBE() { } public int readInt32() { - int readerIdx = readerIndex; // use subtract to avoid overflow int remaining = size - readerIdx; @@ -2333,7 +2220,6 @@ public int readInt32() { // CHECKSTYLE.OFF:MethodName public int _readInt32OnLE() { // CHECKSTYLE.ON:MethodName - int readerIdx = readerIndex; // use subtract to avoid overflow int remaining = size - readerIdx; @@ -2349,7 +2235,6 @@ public int _readInt32OnLE() { // CHECKSTYLE.OFF:MethodName public int _readInt32OnBE() { // CHECKSTYLE.ON:MethodName - int readerIdx = readerIndex; // use subtract to avoid overflow int remaining = size - readerIdx; @@ -2361,7 +2246,6 @@ public int _readInt32OnBE() { } public long readInt64() { - int readerIdx = readerIndex; // use subtract to avoid overflow int remaining = size - readerIdx; @@ -2378,7 +2262,6 @@ public long readInt64() { // CHECKSTYLE.OFF:MethodName public long _readInt64OnLE() { // CHECKSTYLE.ON:MethodName - int readerIdx = readerIndex; // use subtract to avoid overflow int remaining = size - readerIdx; @@ -2394,7 +2277,6 @@ public long _readInt64OnLE() { // CHECKSTYLE.OFF:MethodName public long _readInt64OnBE() { // CHECKSTYLE.ON:MethodName - int readerIdx = readerIndex; // use subtract to avoid overflow int remaining = size - readerIdx; @@ -2427,7 +2309,6 @@ public long readTaggedUInt64() { // CHECKSTYLE.OFF:MethodName public long _readTaggedUInt64OnLE() { // CHECKSTYLE.ON:MethodName - final int readIdx = readerIndex; int diff = size - readIdx; if (diff < 4) { @@ -2450,7 +2331,6 @@ public long _readTaggedUInt64OnLE() { // CHECKSTYLE.OFF:MethodName public long _readTaggedUInt64OnBE() { // CHECKSTYLE.ON:MethodName - final int readIdx = readerIndex; int diff = size - readIdx; if (diff < 4) { @@ -2473,7 +2353,6 @@ public long _readTaggedUInt64OnBE() { // CHECKSTYLE.OFF:MethodName public long _readTaggedInt64OnLE() { // CHECKSTYLE.ON:MethodName - // Duplicate and manual inline for performance. // noinspection Duplicates final int readIdx = readerIndex; @@ -2498,7 +2377,6 @@ public long _readTaggedInt64OnLE() { // CHECKSTYLE.OFF:MethodName public long _readTaggedInt64OnBE() { // CHECKSTYLE.ON:MethodName - // noinspection Duplicates final int readIdx = readerIndex; int diff = size - readIdx; @@ -2519,7 +2397,6 @@ public long _readTaggedInt64OnBE() { } public float readFloat32() { - // noinspection Duplicates int readerIdx = readerIndex; // use subtract to avoid overflow @@ -2540,7 +2417,6 @@ public float readFloat32() { // CHECKSTYLE.OFF:MethodName public float _readFloat32OnLE() { // CHECKSTYLE.ON:MethodName - int readerIdx = readerIndex; // use subtract to avoid overflow int remaining = size - readerIdx; @@ -2556,7 +2432,6 @@ public float _readFloat32OnLE() { // CHECKSTYLE.OFF:MethodName public float _readFloat32OnBE() { // CHECKSTYLE.ON:MethodName - int readerIdx = readerIndex; // use subtract to avoid overflow int remaining = size - readerIdx; @@ -2564,12 +2439,10 @@ public float _readFloat32OnBE() { streamReader.fillBuffer(4 - remaining); } readerIndex = readerIdx + 4; - return Float.intBitsToFloat( - Integer.reverseBytes(loadInt(address + readerIdx))); + return Float.intBitsToFloat(Integer.reverseBytes(loadInt(address + readerIdx))); } public double readFloat64() { - // noinspection Duplicates int readerIdx = readerIndex; // use subtract to avoid overflow @@ -2590,7 +2463,6 @@ public double readFloat64() { // CHECKSTYLE.OFF:MethodName public double _readFloat64OnLE() { // CHECKSTYLE.ON:MethodName - int readerIdx = readerIndex; // use subtract to avoid overflow int remaining = size - readerIdx; @@ -2606,7 +2478,6 @@ public double _readFloat64OnLE() { // CHECKSTYLE.OFF:MethodName public double _readFloat64OnBE() { // CHECKSTYLE.ON:MethodName - int readerIdx = readerIndex; // use subtract to avoid overflow int remaining = size - readerIdx; @@ -2614,14 +2485,12 @@ public double _readFloat64OnBE() { streamReader.fillBuffer(8 - remaining); } readerIndex = readerIdx + 8; - return Double.longBitsToDouble( - Long.reverseBytes(loadLong(address + readerIdx))); + return Double.longBitsToDouble(Long.reverseBytes(loadLong(address + readerIdx))); } /** Reads the 1-5 byte int part of a varint. */ @CodegenInvoke public int readVarInt32() { - if (LITTLE_ENDIAN) { return _readVarInt32OnLE(); } else { @@ -2634,7 +2503,6 @@ public int readVarInt32() { // CHECKSTYLE.OFF:MethodName public int _readVarInt32OnLE() { // CHECKSTYLE.ON:MethodName - // noinspection Duplicates int readIdx = readerIndex; int result; @@ -2682,7 +2550,6 @@ public int _readVarInt32OnLE() { // CHECKSTYLE.OFF:MethodName public int _readVarInt32OnBE() { // CHECKSTYLE.ON:MethodName - // noinspection Duplicates int readIdx = readerIndex; int result; @@ -2725,7 +2592,6 @@ public int _readVarInt32OnBE() { } public long readVarUint36Small() { - // Android exits above. Keep JVM small-varint bulk reads as direct bulk loads instead of calling // `_unsafeGet*` helpers; those helpers carry Android/endian branches and can break inlining. // Duplicate and manual inline for performance. @@ -2835,7 +2701,6 @@ private static void throwMalformedVarUInt32(int fifthByte) { /** Reads the 1-5 byte int part of a non-negative varint. */ public int readVarUInt32() { - int readIdx = readerIndex; if (size - readIdx < 5) { return readVarUInt32Slow(); @@ -2882,7 +2747,6 @@ public int readVarUInt32() { * 127). When the value is equal or greater than 127, the read will be a little slower. */ public int readVarUInt32Small7() { - int readIdx = readerIndex; if (size - readIdx > 0) { byte v = loadByte(address + readIdx++); @@ -2899,7 +2763,6 @@ public int readVarUInt32Small7() { * 16384). When the value is equal or greater than 16384, the read will be a little slower. */ public int readVarUInt32Small14() { - int readIdx = readerIndex; if (size - readIdx >= 5) { int fourByteValue = loadInt(address + readIdx++); @@ -2947,7 +2810,6 @@ private int continueReadVarUInt32(int readIdx, int bulkRead, int value) { /** Reads the 1-9 byte int part of a var long. */ public long readVarInt64() { - return LITTLE_ENDIAN ? _readVarInt64OnLE() : _readVarInt64OnBE(); } @@ -2955,7 +2817,6 @@ public long readVarInt64() { // CHECKSTYLE.OFF:MethodName public long _readVarInt64OnLE() { // CHECKSTYLE.ON:MethodName - // Duplicate and manual inline for performance. // noinspection Duplicates int readIdx = readerIndex; @@ -2988,7 +2849,6 @@ public long _readVarInt64OnLE() { // CHECKSTYLE.OFF:MethodName public long _readVarInt64OnBE() { // CHECKSTYLE.ON:MethodName - int readIdx = readerIndex; long result; if (size - readIdx < 9) { @@ -3017,7 +2877,6 @@ public long _readVarInt64OnBE() { /** Reads the 1-9 byte int part of a non-negative var long. */ public long readVarUInt64() { - int readIdx = readerIndex; if (size - readIdx < 9) { return readVarUInt64Slow(); @@ -3122,7 +2981,6 @@ private long readVarUInt64Slow() { /** Reads the 1-9 byte int part of an aligned varint. */ public int readAlignedVarUInt32() { - int readerIdx = readerIndex; // use subtract to avoid overflow if (readerIdx < size - 10) { @@ -3249,7 +3107,6 @@ public void readBytes(byte[] dst) { /** Read {@code len} bytes into a long using little-endian order. */ public long readBytesAsInt64(int len) { - int readerIdx = readerIndex; // use subtract to avoid overflow int remaining = size - readerIdx; @@ -3319,7 +3176,6 @@ public void read(ByteBuffer dst, int len) { * is optimized for small size, it's faster than {@link #readVarUInt32}. */ public int readBinarySize() { - int binarySize; int readIdx = readerIndex; if (size - readIdx >= 5) { @@ -3396,15 +3252,13 @@ public byte[] readBytesAndSize() { * prefix. */ public void readByteArrayPayload(byte[] values, int numBytes) { - - int readerIdx = readerIndex; - if (readerIdx > size - numBytes) { - streamReader.readTo(values, 0, numBytes); - return; - } - readBytesToArray(address + readerIdx, values, BYTE_ARRAY_OFFSET, numBytes); - readerIndex = readerIdx + numBytes; - + int readerIdx = readerIndex; + if (readerIdx > size - numBytes) { + streamReader.readTo(values, 0, numBytes); + return; + } + readBytesToArray(address + readerIdx, values, BYTE_ARRAY_OFFSET, numBytes); + readerIndex = readerIdx + numBytes; } /** @@ -3413,15 +3267,13 @@ public void readByteArrayPayload(byte[] values, int numBytes) { * prefix. */ public void readBooleanArrayPayload(boolean[] values, int numBytes) { - - int readerIdx = readerIndex; - if (readerIdx > size - numBytes) { - streamReader.readBooleans(values, 0, numBytes); - return; - } - readBooleansToArray(address + readerIdx, values, BOOLEAN_ARRAY_OFFSET, numBytes); - readerIndex = readerIdx + numBytes; - + int readerIdx = readerIndex; + if (readerIdx > size - numBytes) { + streamReader.readBooleans(values, 0, numBytes); + return; + } + readBooleansToArray(address + readerIdx, values, BOOLEAN_ARRAY_OFFSET, numBytes); + readerIndex = readerIdx + numBytes; } /** @@ -3430,15 +3282,13 @@ public void readBooleanArrayPayload(boolean[] values, int numBytes) { * prefix. */ public void readCharArrayPayload(char[] values, int numBytes) { - - int readerIdx = readerIndex; - if (readerIdx > size - numBytes) { - streamReader.readChars(values, 0, numBytes >>> 1); - return; - } - readCharsToArray(address + readerIdx, values, CHAR_ARRAY_OFFSET, numBytes); - readerIndex = readerIdx + numBytes; - + int readerIdx = readerIndex; + if (readerIdx > size - numBytes) { + streamReader.readChars(values, 0, numBytes >>> 1); + return; + } + readCharsToArray(address + readerIdx, values, CHAR_ARRAY_OFFSET, numBytes); + readerIndex = readerIdx + numBytes; } /** @@ -3447,15 +3297,13 @@ public void readCharArrayPayload(char[] values, int numBytes) { * prefix. */ public void readInt16ArrayPayload(short[] values, int numBytes) { - - int readerIdx = readerIndex; - if (readerIdx > size - numBytes) { - streamReader.readShorts(values, 0, numBytes >>> 1); - return; - } - readShortsToArray(address + readerIdx, values, SHORT_ARRAY_OFFSET, numBytes); - readerIndex = readerIdx + numBytes; - + int readerIdx = readerIndex; + if (readerIdx > size - numBytes) { + streamReader.readShorts(values, 0, numBytes >>> 1); + return; + } + readShortsToArray(address + readerIdx, values, SHORT_ARRAY_OFFSET, numBytes); + readerIndex = readerIdx + numBytes; } /** @@ -3464,15 +3312,13 @@ public void readInt16ArrayPayload(short[] values, int numBytes) { * prefix. */ public void readInt32ArrayPayload(int[] values, int numBytes) { - - int readerIdx = readerIndex; - if (readerIdx > size - numBytes) { - streamReader.readInts(values, 0, numBytes >>> 2); - return; - } - readIntsToArray(address + readerIdx, values, INT_ARRAY_OFFSET, numBytes); - readerIndex = readerIdx + numBytes; - + int readerIdx = readerIndex; + if (readerIdx > size - numBytes) { + streamReader.readInts(values, 0, numBytes >>> 2); + return; + } + readIntsToArray(address + readerIdx, values, INT_ARRAY_OFFSET, numBytes); + readerIndex = readerIdx + numBytes; } /** @@ -3481,15 +3327,13 @@ public void readInt32ArrayPayload(int[] values, int numBytes) { * prefix. */ public void readInt64ArrayPayload(long[] values, int numBytes) { - - int readerIdx = readerIndex; - if (readerIdx > size - numBytes) { - streamReader.readLongs(values, 0, numBytes >>> 3); - return; - } - readLongsToArray(address + readerIdx, values, LONG_ARRAY_OFFSET, numBytes); - readerIndex = readerIdx + numBytes; - + int readerIdx = readerIndex; + if (readerIdx > size - numBytes) { + streamReader.readLongs(values, 0, numBytes >>> 3); + return; + } + readLongsToArray(address + readerIdx, values, LONG_ARRAY_OFFSET, numBytes); + readerIndex = readerIdx + numBytes; } /** @@ -3498,15 +3342,13 @@ public void readInt64ArrayPayload(long[] values, int numBytes) { * prefix. */ public void readFloat32ArrayPayload(float[] values, int numBytes) { - - int readerIdx = readerIndex; - if (readerIdx > size - numBytes) { - streamReader.readFloats(values, 0, numBytes >>> 2); - return; - } - readFloatsToArray(address + readerIdx, values, FLOAT_ARRAY_OFFSET, numBytes); - readerIndex = readerIdx + numBytes; - + int readerIdx = readerIndex; + if (readerIdx > size - numBytes) { + streamReader.readFloats(values, 0, numBytes >>> 2); + return; + } + readFloatsToArray(address + readerIdx, values, FLOAT_ARRAY_OFFSET, numBytes); + readerIndex = readerIdx + numBytes; } /** @@ -3515,31 +3357,27 @@ public void readFloat32ArrayPayload(float[] values, int numBytes) { * prefix. */ public void readFloat64ArrayPayload(double[] values, int numBytes) { - - int readerIdx = readerIndex; - if (readerIdx > size - numBytes) { - streamReader.readDoubles(values, 0, numBytes >>> 3); - return; - } - readDoublesToArray(address + readerIdx, values, DOUBLE_ARRAY_OFFSET, numBytes); - readerIndex = readerIdx + numBytes; - + int readerIdx = readerIndex; + if (readerIdx > size - numBytes) { + streamReader.readDoubles(values, 0, numBytes >>> 3); + return; + } + readDoublesToArray(address + readerIdx, values, DOUBLE_ARRAY_OFFSET, numBytes); + readerIndex = readerIdx + numBytes; } public void readBooleans(boolean[] values, int offset, int numElements) { - - if ((offset | numElements | (offset + numElements) | (values.length - offset - numElements)) - < 0) { - throwOOBException(); - } - if (readerIndex > size - numElements) { - streamReader.readBooleans(values, offset, numElements); - return; - } - int readerIdx = readerIndex; - readBooleansToArray(address + readerIdx, values, BOOLEAN_ARRAY_OFFSET + offset, numElements); - readerIndex = readerIdx + numElements; - + if ((offset | numElements | (offset + numElements) | (values.length - offset - numElements)) + < 0) { + throwOOBException(); + } + if (readerIndex > size - numElements) { + streamReader.readBooleans(values, offset, numElements); + return; + } + int readerIdx = readerIndex; + readBooleansToArray(address + readerIdx, values, BOOLEAN_ARRAY_OFFSET + offset, numElements); + readerIndex = readerIdx + numElements; } public void readChars(char[] chars, int numElements) { @@ -3547,20 +3385,18 @@ public void readChars(char[] chars, int numElements) { } public void readChars(char[] chars, int offset, int numElements) { - - if ((offset | numElements | (offset + numElements) | (chars.length - offset - numElements)) - < 0) { - throwOOBException(); - } - int numBytes = Math.multiplyExact(numElements, 2); - if (readerIndex > size - numBytes) { - streamReader.readChars(chars, offset, numElements); - return; - } - int readerIdx = readerIndex; - readCharsToArray(address + readerIdx, chars, CHAR_ARRAY_OFFSET + offset, numBytes); - readerIndex = readerIdx + numBytes; - + if ((offset | numElements | (offset + numElements) | (chars.length - offset - numElements)) + < 0) { + throwOOBException(); + } + int numBytes = Math.multiplyExact(numElements, 2); + if (readerIndex > size - numBytes) { + streamReader.readChars(chars, offset, numElements); + return; + } + int readerIdx = readerIndex; + readCharsToArray(address + readerIdx, chars, CHAR_ARRAY_OFFSET + offset, numBytes); + readerIndex = readerIdx + numBytes; } @CodegenInvoke @@ -3573,88 +3409,78 @@ public char[] readCharsAndSize() { } public void readShorts(short[] values, int offset, int numElements) { - - if ((offset | numElements | (offset + numElements) | (values.length - offset - numElements)) - < 0) { - throwOOBException(); - } - int numBytes = Math.multiplyExact(numElements, 2); - if (readerIndex > size - numBytes) { - streamReader.readShorts(values, offset, numElements); - return; - } - int readerIdx = readerIndex; - readShortsToArray(address + readerIdx, values, SHORT_ARRAY_OFFSET + offset, numBytes); - readerIndex = readerIdx + numBytes; - + if ((offset | numElements | (offset + numElements) | (values.length - offset - numElements)) + < 0) { + throwOOBException(); + } + int numBytes = Math.multiplyExact(numElements, 2); + if (readerIndex > size - numBytes) { + streamReader.readShorts(values, offset, numElements); + return; + } + int readerIdx = readerIndex; + readShortsToArray(address + readerIdx, values, SHORT_ARRAY_OFFSET + offset, numBytes); + readerIndex = readerIdx + numBytes; } public void readInts(int[] values, int offset, int numElements) { - - if ((offset | numElements | (offset + numElements) | (values.length - offset - numElements)) - < 0) { - throwOOBException(); - } - int numBytes = Math.multiplyExact(numElements, 4); - if (readerIndex > size - numBytes) { - streamReader.readInts(values, offset, numElements); - return; - } - int readerIdx = readerIndex; - readIntsToArray(address + readerIdx, values, INT_ARRAY_OFFSET + offset, numBytes); - readerIndex = readerIdx + numBytes; - + if ((offset | numElements | (offset + numElements) | (values.length - offset - numElements)) + < 0) { + throwOOBException(); + } + int numBytes = Math.multiplyExact(numElements, 4); + if (readerIndex > size - numBytes) { + streamReader.readInts(values, offset, numElements); + return; + } + int readerIdx = readerIndex; + readIntsToArray(address + readerIdx, values, INT_ARRAY_OFFSET + offset, numBytes); + readerIndex = readerIdx + numBytes; } public void readLongs(long[] values, int offset, int numElements) { - - if ((offset | numElements | (offset + numElements) | (values.length - offset - numElements)) - < 0) { - throwOOBException(); - } - int numBytes = Math.multiplyExact(numElements, 8); - if (readerIndex > size - numBytes) { - streamReader.readLongs(values, offset, numElements); - return; - } - int readerIdx = readerIndex; - readLongsToArray(address + readerIdx, values, LONG_ARRAY_OFFSET + offset, numBytes); - readerIndex = readerIdx + numBytes; - + if ((offset | numElements | (offset + numElements) | (values.length - offset - numElements)) + < 0) { + throwOOBException(); + } + int numBytes = Math.multiplyExact(numElements, 8); + if (readerIndex > size - numBytes) { + streamReader.readLongs(values, offset, numElements); + return; + } + int readerIdx = readerIndex; + readLongsToArray(address + readerIdx, values, LONG_ARRAY_OFFSET + offset, numBytes); + readerIndex = readerIdx + numBytes; } public void readFloats(float[] values, int offset, int numElements) { - - if ((offset | numElements | (offset + numElements) | (values.length - offset - numElements)) - < 0) { - throwOOBException(); - } - int numBytes = Math.multiplyExact(numElements, 4); - if (readerIndex > size - numBytes) { - streamReader.readFloats(values, offset, numElements); - return; - } - int readerIdx = readerIndex; - readFloatsToArray(address + readerIdx, values, FLOAT_ARRAY_OFFSET + offset, numBytes); - readerIndex = readerIdx + numBytes; - + if ((offset | numElements | (offset + numElements) | (values.length - offset - numElements)) + < 0) { + throwOOBException(); + } + int numBytes = Math.multiplyExact(numElements, 4); + if (readerIndex > size - numBytes) { + streamReader.readFloats(values, offset, numElements); + return; + } + int readerIdx = readerIndex; + readFloatsToArray(address + readerIdx, values, FLOAT_ARRAY_OFFSET + offset, numBytes); + readerIndex = readerIdx + numBytes; } public void readDoubles(double[] values, int offset, int numElements) { - - if ((offset | numElements | (offset + numElements) | (values.length - offset - numElements)) - < 0) { - throwOOBException(); - } - int numBytes = Math.multiplyExact(numElements, 8); - if (readerIndex > size - numBytes) { - streamReader.readDoubles(values, offset, numElements); - return; - } - int readerIdx = readerIndex; - readDoublesToArray(address + readerIdx, values, DOUBLE_ARRAY_OFFSET + offset, numBytes); - readerIndex = readerIdx + numBytes; - + if ((offset | numElements | (offset + numElements) | (values.length - offset - numElements)) + < 0) { + throwOOBException(); + } + int numBytes = Math.multiplyExact(numElements, 8); + if (readerIndex > size - numBytes) { + streamReader.readDoubles(values, offset, numElements); + return; + } + int readerIdx = readerIndex; + readDoublesToArray(address + readerIdx, values, DOUBLE_ARRAY_OFFSET + offset, numBytes); + readerIndex = readerIdx + numBytes; } public void checkReadableBytes(int minimumReadableBytes) { @@ -3681,30 +3507,29 @@ public byte[] getRemainingBytes() { // ------------------------- Read Methods Finished ------------------------------------- public void copyTo(int offset, MemoryBuffer target, int targetOffset, int numBytes) { - - final byte[] thisHeapRef = this.heapMemory; - final byte[] otherHeapRef = target.heapMemory; - final long thisPointer = this.address + offset; - final long otherPointer = target.address + targetOffset; - if ((numBytes | offset | targetOffset) >= 0 - && thisPointer <= this.addressLimit - numBytes - && otherPointer <= target.addressLimit - numBytes) { - if (thisHeapRef != null && otherHeapRef != null) { - System.arraycopy(thisHeapRef, toIntIndex(thisPointer), otherHeapRef, toIntIndex(otherPointer), numBytes); - } else if (sameBufferOverlap(target, offset, targetOffset, numBytes)) { - byte[] tmp = new byte[numBytes]; - sliceAsByteBuffer(offset, numBytes).get(tmp); - target.sliceAsByteBuffer(targetOffset, numBytes).put(tmp); - } else { - target.sliceAsByteBuffer(targetOffset, numBytes).put(sliceAsByteBuffer(offset, numBytes)); - } + final byte[] thisHeapRef = this.heapMemory; + final byte[] otherHeapRef = target.heapMemory; + final long thisPointer = this.address + offset; + final long otherPointer = target.address + targetOffset; + if ((numBytes | offset | targetOffset) >= 0 + && thisPointer <= this.addressLimit - numBytes + && otherPointer <= target.addressLimit - numBytes) { + if (thisHeapRef != null && otherHeapRef != null) { + System.arraycopy( + thisHeapRef, toIntIndex(thisPointer), otherHeapRef, toIntIndex(otherPointer), numBytes); + } else if (sameBufferOverlap(target, offset, targetOffset, numBytes)) { + byte[] tmp = new byte[numBytes]; + sliceAsByteBuffer(offset, numBytes).get(tmp); + target.sliceAsByteBuffer(targetOffset, numBytes).put(tmp); } else { - throw new IndexOutOfBoundsException( - String.format( - "offset=%d, targetOffset=%d, numBytes=%d, address=%d, targetAddress=%d", - offset, targetOffset, numBytes, this.address, target.address)); + target.sliceAsByteBuffer(targetOffset, numBytes).put(sliceAsByteBuffer(offset, numBytes)); } - + } else { + throw new IndexOutOfBoundsException( + String.format( + "offset=%d, targetOffset=%d, numBytes=%d, address=%d, targetAddress=%d", + offset, targetOffset, numBytes, this.address, target.address)); + } } private boolean sameBufferOverlap( @@ -3730,59 +3555,43 @@ public void copyFrom(int offset, MemoryBuffer source, int sourcePointer, int num } public void copyToByteArray(int offset, byte[] target, int targetOffset, int numBytes) { - - checkArrayCopy(offset, targetOffset, target.length, numBytes, 0); - readBytesToArray(address + offset, target, BYTE_ARRAY_OFFSET + targetOffset, numBytes); - + checkArrayCopy(offset, targetOffset, target.length, numBytes, 0); + readBytesToArray(address + offset, target, BYTE_ARRAY_OFFSET + targetOffset, numBytes); } public void copyToBooleanArray(int offset, boolean[] target, int targetOffset, int numBytes) { - - checkArrayCopy(offset, targetOffset, target.length, numBytes, 0); - readBooleansToArray(address + offset, target, BOOLEAN_ARRAY_OFFSET + targetOffset, numBytes); - + checkArrayCopy(offset, targetOffset, target.length, numBytes, 0); + readBooleansToArray(address + offset, target, BOOLEAN_ARRAY_OFFSET + targetOffset, numBytes); } public void copyToCharArray(int offset, char[] target, int targetOffset, int numBytes) { - - checkArrayCopy(offset, targetOffset, target.length, numBytes, 1); - readCharsToArray(address + offset, target, CHAR_ARRAY_OFFSET + targetOffset, numBytes); - + checkArrayCopy(offset, targetOffset, target.length, numBytes, 1); + readCharsToArray(address + offset, target, CHAR_ARRAY_OFFSET + targetOffset, numBytes); } public void copyToShortArray(int offset, short[] target, int targetOffset, int numBytes) { - - checkArrayCopy(offset, targetOffset, target.length, numBytes, 1); - readShortsToArray(address + offset, target, SHORT_ARRAY_OFFSET + targetOffset, numBytes); - + checkArrayCopy(offset, targetOffset, target.length, numBytes, 1); + readShortsToArray(address + offset, target, SHORT_ARRAY_OFFSET + targetOffset, numBytes); } public void copyToIntArray(int offset, int[] target, int targetOffset, int numBytes) { - - checkArrayCopy(offset, targetOffset, target.length, numBytes, 2); - readIntsToArray(address + offset, target, INT_ARRAY_OFFSET + targetOffset, numBytes); - + checkArrayCopy(offset, targetOffset, target.length, numBytes, 2); + readIntsToArray(address + offset, target, INT_ARRAY_OFFSET + targetOffset, numBytes); } public void copyToLongArray(int offset, long[] target, int targetOffset, int numBytes) { - - checkArrayCopy(offset, targetOffset, target.length, numBytes, 3); - readLongsToArray(address + offset, target, LONG_ARRAY_OFFSET + targetOffset, numBytes); - + checkArrayCopy(offset, targetOffset, target.length, numBytes, 3); + readLongsToArray(address + offset, target, LONG_ARRAY_OFFSET + targetOffset, numBytes); } public void copyToFloatArray(int offset, float[] target, int targetOffset, int numBytes) { - - checkArrayCopy(offset, targetOffset, target.length, numBytes, 2); - readFloatsToArray(address + offset, target, FLOAT_ARRAY_OFFSET + targetOffset, numBytes); - + checkArrayCopy(offset, targetOffset, target.length, numBytes, 2); + readFloatsToArray(address + offset, target, FLOAT_ARRAY_OFFSET + targetOffset, numBytes); } public void copyToDoubleArray(int offset, double[] target, int targetOffset, int numBytes) { - - checkArrayCopy(offset, targetOffset, target.length, numBytes, 3); - readDoublesToArray(address + offset, target, DOUBLE_ARRAY_OFFSET + targetOffset, numBytes); - + checkArrayCopy(offset, targetOffset, target.length, numBytes, 3); + readDoublesToArray(address + offset, target, DOUBLE_ARRAY_OFFSET + targetOffset, numBytes); } private void checkArrayCopy( @@ -3800,60 +3609,43 @@ private void checkArrayCopy( } public void copyFromByteArray(int offset, byte[] source, int sourceOffset, int numBytes) { - - checkArrayCopy(offset, sourceOffset, source.length, numBytes, 0); - writeBytesFromArray(address + offset, source, BYTE_ARRAY_OFFSET + sourceOffset, numBytes); - + checkArrayCopy(offset, sourceOffset, source.length, numBytes, 0); + writeBytesFromArray(address + offset, source, BYTE_ARRAY_OFFSET + sourceOffset, numBytes); } public void copyFromBooleanArray(int offset, boolean[] source, int sourceOffset, int numBytes) { - - checkArrayCopy(offset, sourceOffset, source.length, numBytes, 0); - writeBooleansFromArray( - address + offset, source, BOOLEAN_ARRAY_OFFSET + sourceOffset, numBytes); - + checkArrayCopy(offset, sourceOffset, source.length, numBytes, 0); + writeBooleansFromArray(address + offset, source, BOOLEAN_ARRAY_OFFSET + sourceOffset, numBytes); } public void copyFromCharArray(int offset, char[] source, int sourceOffset, int numBytes) { - - checkArrayCopy(offset, sourceOffset, source.length, numBytes, 1); - writeCharsFromArray(address + offset, source, CHAR_ARRAY_OFFSET + sourceOffset, numBytes); - + checkArrayCopy(offset, sourceOffset, source.length, numBytes, 1); + writeCharsFromArray(address + offset, source, CHAR_ARRAY_OFFSET + sourceOffset, numBytes); } public void copyFromShortArray(int offset, short[] source, int sourceOffset, int numBytes) { - - checkArrayCopy(offset, sourceOffset, source.length, numBytes, 1); - writeShortsFromArray(address + offset, source, SHORT_ARRAY_OFFSET + sourceOffset, numBytes); - + checkArrayCopy(offset, sourceOffset, source.length, numBytes, 1); + writeShortsFromArray(address + offset, source, SHORT_ARRAY_OFFSET + sourceOffset, numBytes); } public void copyFromIntArray(int offset, int[] source, int sourceOffset, int numBytes) { - - checkArrayCopy(offset, sourceOffset, source.length, numBytes, 2); - writeIntsFromArray(address + offset, source, INT_ARRAY_OFFSET + sourceOffset, numBytes); - + checkArrayCopy(offset, sourceOffset, source.length, numBytes, 2); + writeIntsFromArray(address + offset, source, INT_ARRAY_OFFSET + sourceOffset, numBytes); } public void copyFromLongArray(int offset, long[] source, int sourceOffset, int numBytes) { - - checkArrayCopy(offset, sourceOffset, source.length, numBytes, 3); - writeLongsFromArray(address + offset, source, LONG_ARRAY_OFFSET + sourceOffset, numBytes); - + checkArrayCopy(offset, sourceOffset, source.length, numBytes, 3); + writeLongsFromArray(address + offset, source, LONG_ARRAY_OFFSET + sourceOffset, numBytes); } public void copyFromFloatArray(int offset, float[] source, int sourceOffset, int numBytes) { - - checkArrayCopy(offset, sourceOffset, source.length, numBytes, 2); - writeFloatsFromArray(address + offset, source, FLOAT_ARRAY_OFFSET + sourceOffset, numBytes); - + checkArrayCopy(offset, sourceOffset, source.length, numBytes, 2); + writeFloatsFromArray(address + offset, source, FLOAT_ARRAY_OFFSET + sourceOffset, numBytes); } public void copyFromDoubleArray(int offset, double[] source, int sourceOffset, int numBytes) { - - checkArrayCopy(offset, sourceOffset, source.length, numBytes, 3); - writeDoublesFromArray(address + offset, source, DOUBLE_ARRAY_OFFSET + sourceOffset, numBytes); - + checkArrayCopy(offset, sourceOffset, source.length, numBytes, 3); + writeDoublesFromArray(address + offset, source, DOUBLE_ARRAY_OFFSET + sourceOffset, numBytes); } public byte[] getBytes(int index, int length) { @@ -4135,7 +3927,6 @@ public static MemoryBuffer fromByteBuffer(ByteBuffer buffer) { public static MemoryBuffer fromDirectByteBuffer( ByteBuffer buffer, int size, ForyStreamReader streamReader) { - long offHeapAddress = buffer.position(); return new MemoryBuffer(offHeapAddress, size, buffer, streamReader); } diff --git a/java/fory-core/src/main/java25/org/apache/fory/platform/internal/DefineClass.java b/java/fory-core/src/main/java25/org/apache/fory/platform/internal/DefineClass.java index 5afebee02f..5214658cfc 100644 --- a/java/fory-core/src/main/java25/org/apache/fory/platform/internal/DefineClass.java +++ b/java/fory-core/src/main/java25/org/apache/fory/platform/internal/DefineClass.java @@ -80,9 +80,7 @@ public static Class defineHiddenNestmate(Class neighbor, byte[] bytecodes) Preconditions.checkNotNull(bytecodes); try { Lookup lookup = _Lookup.privateLookupIn(neighbor, MethodHandles.lookup()); - return lookup - .defineHiddenClass(bytecodes, true, Lookup.ClassOption.NESTMATE) - .lookupClass(); + return lookup.defineHiddenClass(bytecodes, true, Lookup.ClassOption.NESTMATE).lookupClass(); } catch (IllegalAccessException | IllegalStateException e) { throw new IllegalStateException( "Cannot define hidden nestmate for " diff --git a/java/fory-core/src/main/java25/org/apache/fory/serializer/SerializationHookLookup.java b/java/fory-core/src/main/java25/org/apache/fory/serializer/SerializationHookLookup.java index f94aa161fc..bd756794f8 100644 --- a/java/fory-core/src/main/java25/org/apache/fory/serializer/SerializationHookLookup.java +++ b/java/fory-core/src/main/java25/org/apache/fory/serializer/SerializationHookLookup.java @@ -33,7 +33,8 @@ final class SerializationHookLookup { private SerializationHookLookup() {} - static MethodHandle readResolveHandle(Class type, Method method) throws IllegalAccessException { + static MethodHandle readResolveHandle(Class type, Method method) + throws IllegalAccessException { try { return _JDKAccess._trustedLookup(type).unreflect(method); } catch (IllegalArgumentException e) { From 313035a40664239a9a27dc4f27c6a599e1e5fb74 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Tue, 2 Jun 2026 14:55:13 +0800 Subject: [PATCH 43/69] refactor(java): clean jdk25 hook lookups --- .agents/languages/java.md | 6 + benchmarks/java/pom.xml | 8 - .../fory/benchmark/Jdk25MrJarCheck.java | 44 ++++-- java/fory-core/pom.xml | 8 - .../serializer/ObjectStreamSerializer.java | 9 -- .../serializer/ReplaceResolveSerializer.java | 9 +- .../serializer/SerializationHookLookup.java | 144 ++++++++++++++---- .../org/apache/fory/memory/LittleEndian.java | 25 ++- .../fory/serializer/PlatformStringUtils.java | 22 --- .../serializer/SerializationHookLookup.java | 144 ------------------ 10 files changed, 161 insertions(+), 258 deletions(-) delete mode 100644 java/fory-core/src/main/java25/org/apache/fory/serializer/SerializationHookLookup.java diff --git a/.agents/languages/java.md b/.agents/languages/java.md index 1d659fa984..54e22012a0 100644 --- a/.agents/languages/java.md +++ b/.agents/languages/java.md @@ -83,6 +83,12 @@ Load this file when changing anything under `java/` or when Java drives a cross- focused on field and array access, keep serialization hook discovery in serializer-owned code, and keep `_JDKAccess` limited to JDK lookup, module, function factory, and access-flag primitives. +- JDK25+ serialization hook access must use the required trusted lookup from + `java.base/java.lang.invoke=org.apache.fory.core`. Keep `sun.reflect.ReflectionFactory` as a + JDK8-24 optimization only, and do not add per-type reflective escapes for hook invocation. +- JDK25+ `PlatformStringUtils` getter methods sit behind `StringSerializer` static-final access + gates. Do not add per-call access checks in those getters; missing module opens should fail at + trusted-lookup initialization or cold setup, not inside string hot paths. - JDK25+ collection serializers must fail unsupported `Collections.newSetFromMap` backing maps before writing or copying. Do not rewrite them to `HashMap`, because that changes equality semantics and can drop entries. diff --git a/benchmarks/java/pom.xml b/benchmarks/java/pom.xml index c85fde3e03..a7bf23af8a 100644 --- a/benchmarks/java/pom.xml +++ b/benchmarks/java/pom.xml @@ -265,8 +265,6 @@ name="META-INF/versions/25/org/apache/fory/memory/LittleEndian.class"/> - - @@ -311,9 +306,6 @@ - diff --git a/benchmarks/java/src/main/java/org/apache/fory/benchmark/Jdk25MrJarCheck.java b/benchmarks/java/src/main/java/org/apache/fory/benchmark/Jdk25MrJarCheck.java index 4890268b0b..b1a50b5304 100644 --- a/benchmarks/java/src/main/java/org/apache/fory/benchmark/Jdk25MrJarCheck.java +++ b/benchmarks/java/src/main/java/org/apache/fory/benchmark/Jdk25MrJarCheck.java @@ -26,13 +26,13 @@ public final class Jdk25MrJarCheck { private Jdk25MrJarCheck() {} public static void main(String[] args) { - verifyClass(MemoryBuffer.class); + verifyVersionedClass(MemoryBuffer.class); verifyMissing("org.apache.fory.platform.UnsafeOps"); - Class jdkAccess = verifyClass("org.apache.fory.platform.internal._JDKAccess"); - verifyClass("org.apache.fory.reflect.FieldAccessorStrategy"); - verifyClass("org.apache.fory.serializer.PlatformStringUtils"); - if (getUnsafeField(jdkAccess) != null) { - throw new IllegalStateException("JDK25 benchmark jar loaded Unsafe-backed _JDKAccess"); + Class jdkAccess = verifyRootClass("org.apache.fory.platform.internal._JDKAccess"); + verifyVersionedClass("org.apache.fory.reflect.FieldAccessorStrategy"); + verifyVersionedClass("org.apache.fory.serializer.PlatformStringUtils"); + if (hasUnsafeField(jdkAccess)) { + throw new IllegalStateException("JDK25 benchmark jar loaded Unsafe-owning _JDKAccess"); } } @@ -48,14 +48,19 @@ private static void verifyMissing(String className) { private static Class verifyClass(String className) { try { Class cls = Class.forName(className); - verifyClass(cls); return cls; } catch (ClassNotFoundException e) { throw new IllegalStateException("JDK25 benchmark jar is missing " + className, e); } } - private static void verifyClass(Class cls) { + private static Class verifyVersionedClass(String className) { + Class cls = verifyClass(className); + verifyVersionedClass(cls); + return cls; + } + + private static void verifyVersionedClass(Class cls) { String resourceName = cls.getSimpleName() + ".class"; String resource = String.valueOf(cls.getResource(resourceName)); if (!resource.contains("benchmarks.jar!") || !resource.contains("!/META-INF/versions/25/")) { @@ -63,13 +68,22 @@ private static void verifyClass(Class cls) { } } - private static Object getUnsafeField(Class jdkAccess) { - try { - return jdkAccess.getField("UNSAFE").get(null); - } catch (NoSuchFieldException expected) { - return null; - } catch (ReflectiveOperationException e) { - throw new IllegalStateException("Failed to inspect _JDKAccess.UNSAFE", e); + private static Class verifyRootClass(String className) { + Class cls = verifyClass(className); + String resourceName = cls.getSimpleName() + ".class"; + String resource = String.valueOf(cls.getResource(resourceName)); + if (!resource.contains("benchmarks.jar!") || resource.contains("!/META-INF/versions/25/")) { + throw new IllegalStateException("JDK25 benchmark jar loaded versioned class for " + cls); + } + return cls; + } + + private static boolean hasUnsafeField(Class jdkAccess) { + for (java.lang.reflect.Field field : jdkAccess.getDeclaredFields()) { + if (field.getName().equals("UNSAFE") || field.getType().getName().equals("sun.misc.Unsafe")) { + return true; + } } + return false; } } diff --git a/java/fory-core/pom.xml b/java/fory-core/pom.xml index 5783349a10..bf8ccb47f0 100644 --- a/java/fory-core/pom.xml +++ b/java/fory-core/pom.xml @@ -438,8 +438,6 @@ - @@ -463,9 +461,6 @@ - @@ -502,9 +497,6 @@ - diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectStreamSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectStreamSerializer.java index 31c0b3c488..a72c1e78dc 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectStreamSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectStreamSerializer.java @@ -697,15 +697,6 @@ private StreamTypeInfo(Class type) { writeMethod = SerializationHookLookup.getWriteObjectMethod(type); readMethod = SerializationHookLookup.getReadObjectMethod(type); noDataMethod = SerializationHookLookup.getReadObjectNoDataMethod(type); - if (writeMethod == null) { - writeMethod = JavaSerializer.getWriteObjectMethod(type, false); - } - if (readMethod == null) { - readMethod = JavaSerializer.getReadRefMethod(type, false); - } - if (noDataMethod == null) { - noDataMethod = JavaSerializer.getReadRefNoData(type, false); - } } this.writeObjectMethod = writeMethod; this.readObjectMethod = readMethod; diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/ReplaceResolveSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/ReplaceResolveSerializer.java index ea1a0bcfb8..eedd48052b 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/ReplaceResolveSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/ReplaceResolveSerializer.java @@ -76,13 +76,8 @@ private ReplaceResolveInfo(Class cls) { writeReplaceMethod = JavaSerializer.getWriteReplaceMethod(cls); readResolveMethod = JavaSerializer.getReadResolveMethod(cls); } else if (Serializable.class.isAssignableFrom(cls)) { - if (SerializationHookLookup.isAvailable()) { - writeReplaceMethod = SerializationHookLookup.getWriteReplaceMethod(cls); - readResolveMethod = SerializationHookLookup.getReadResolveMethod(cls); - } else { - writeReplaceMethod = JavaSerializer.getWriteReplaceMethod(cls); - readResolveMethod = JavaSerializer.getReadResolveMethod(cls); - } + writeReplaceMethod = SerializationHookLookup.getWriteReplaceMethod(cls); + readResolveMethod = SerializationHookLookup.getReadResolveMethod(cls); } else { // FIXME class with `writeReplace` method defined should be Serializable, // but hessian ignores this check and many existing system are using hessian, diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/SerializationHookLookup.java b/java/fory-core/src/main/java/org/apache/fory/serializer/SerializationHookLookup.java index d6e4efaa09..6cbfe2e1e2 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/SerializationHookLookup.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/SerializationHookLookup.java @@ -19,9 +19,14 @@ package org.apache.fory.serializer; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; import java.lang.invoke.MethodHandle; import java.lang.invoke.MethodHandles; import java.lang.reflect.Method; +import java.lang.reflect.Modifier; +import org.apache.fory.collection.ClassValueCache; +import org.apache.fory.platform.JdkVersion; import org.apache.fory.platform.internal._JDKAccess; import org.apache.fory.util.ExceptionUtils; @@ -50,24 +55,27 @@ private static final class Methods { Method defaultReadObject = null; Method writeReplace = null; Method readResolve = null; - try { - Class factoryClass = Class.forName("sun.reflect.ReflectionFactory"); - Method getReflectionFactory = factoryClass.getDeclaredMethod("getReflectionFactory"); - reflectionFactory = getReflectionFactory.invoke(null); - writeObject = factoryClass.getDeclaredMethod("writeObjectForSerialization", Class.class); - readObject = factoryClass.getDeclaredMethod("readObjectForSerialization", Class.class); - readObjectNoData = - factoryClass.getDeclaredMethod("readObjectNoDataForSerialization", Class.class); + if (JdkVersion.MAJOR_VERSION < 25) { try { - defaultReadObject = - factoryClass.getDeclaredMethod("defaultReadObjectForSerialization", Class.class); - } catch (NoSuchMethodException e) { + Class factoryClass = Class.forName("sun.reflect.ReflectionFactory"); + Method getReflectionFactory = factoryClass.getDeclaredMethod("getReflectionFactory"); + reflectionFactory = getReflectionFactory.invoke(null); + writeObject = factoryClass.getDeclaredMethod("writeObjectForSerialization", Class.class); + readObject = factoryClass.getDeclaredMethod("readObjectForSerialization", Class.class); + readObjectNoData = + factoryClass.getDeclaredMethod("readObjectNoDataForSerialization", Class.class); + try { + defaultReadObject = + factoryClass.getDeclaredMethod("defaultReadObjectForSerialization", Class.class); + } catch (NoSuchMethodException e) { + ExceptionUtils.ignore(e); + } + writeReplace = + factoryClass.getDeclaredMethod("writeReplaceForSerialization", Class.class); + readResolve = factoryClass.getDeclaredMethod("readResolveForSerialization", Class.class); + } catch (Throwable e) { ExceptionUtils.ignore(e); } - writeReplace = factoryClass.getDeclaredMethod("writeReplaceForSerialization", Class.class); - readResolve = factoryClass.getDeclaredMethod("readResolveForSerialization", Class.class); - } catch (Throwable e) { - ExceptionUtils.ignore(e); } REFLECTION_FACTORY = reflectionFactory; WRITE_OBJECT = writeObject; @@ -96,16 +104,101 @@ private static Method getMethod(Class type, Method factoryMethod) { return handle == null ? null : MethodHandles.reflectAs(Method.class, handle); } + private static final ClassValueCache directMethodsCache = + ClassValueCache.newClassKeyCache(32); + + private static DirectMethods directMethods(Class type) { + // JavaSerializer intentionally supports non-Serializable compatibility hooks. Serializable + // stream hooks must keep ObjectStreamClass inheritance rules, especially for private + // superclass writeReplace/readResolve methods. + return directMethodsCache.get(type, () -> new DirectMethods(type)); + } + + private static final class DirectMethods { + private final Method writeObject; + private final Method readObject; + private final Method readObjectNoData; + private final Method writeReplace; + private final Method readResolve; + + private DirectMethods(Class type) { + writeObject = getPrivateMethod(type, "writeObject", void.class, ObjectOutputStream.class); + readObject = getPrivateMethod(type, "readObject", void.class, ObjectInputStream.class); + readObjectNoData = getPrivateMethod(type, "readObjectNoData", void.class); + writeReplace = getInheritableObjectMethod(type, "writeReplace"); + readResolve = getInheritableObjectMethod(type, "readResolve"); + } + } + + private static Method getPrivateMethod( + Class type, String methodName, Class returnType, Class... parameterTypes) { + try { + Method method = type.getDeclaredMethod(methodName, parameterTypes); + int modifiers = method.getModifiers(); + if (Modifier.isPrivate(modifiers) + && !Modifier.isStatic(modifiers) + && method.getReturnType() == returnType) { + return method; + } + } catch (NoSuchMethodException | SecurityException e) { + ExceptionUtils.ignore(e); + } + return null; + } + + private static Method getInheritableObjectMethod(Class type, String methodName) { + Class cls = type; + while (cls != null) { + try { + Method method = cls.getDeclaredMethod(methodName); + int modifiers = method.getModifiers(); + if (!Modifier.isStatic(modifiers) + && method.getParameterTypes().length == 0 + && method.getReturnType() == Object.class + && isInheritable(type, cls, modifiers)) { + return method; + } + return null; + } catch (NoSuchMethodException e) { + ExceptionUtils.ignore(e); + } catch (SecurityException e) { + return null; + } + cls = cls.getSuperclass(); + } + return null; + } + + private static boolean isInheritable(Class type, Class declaringClass, int modifiers) { + if (Modifier.isPrivate(modifiers)) { + return type == declaringClass; + } + if (Modifier.isPublic(modifiers) || Modifier.isProtected(modifiers)) { + return true; + } + return type.getClassLoader() == declaringClass.getClassLoader() + && packageName(type).equals(packageName(declaringClass)); + } + + private static String packageName(Class type) { + String className = type.getName(); + int packageEnd = className.lastIndexOf('.'); + return packageEnd < 0 ? "" : className.substring(0, packageEnd); + } + static Method getWriteObjectMethod(Class type) { - return getMethod(type, Methods.WRITE_OBJECT); + Method method = getMethod(type, Methods.WRITE_OBJECT); + return method == null ? directMethods(type).writeObject : method; } static Method getReadObjectMethod(Class type) { - return getMethod(type, Methods.READ_OBJECT); + Method method = getMethod(type, Methods.READ_OBJECT); + return method == null ? directMethods(type).readObject : method; } static Method getReadObjectNoDataMethod(Class type) { - return getMethod(type, Methods.READ_OBJECT_NO_DATA); + Method method = getMethod(type, Methods.READ_OBJECT_NO_DATA); + return method == null ? directMethods(type).readObjectNoData : method; } static MethodHandle getDefaultReadObjectHandle(Class type) { @@ -113,19 +206,12 @@ static MethodHandle getDefaultReadObjectHandle(Class type) { } static Method getWriteReplaceMethod(Class type) { - return getMethod(type, Methods.WRITE_REPLACE); + Method method = getMethod(type, Methods.WRITE_REPLACE); + return method == null ? directMethods(type).writeReplace : method; } static Method getReadResolveMethod(Class type) { - return getMethod(type, Methods.READ_RESOLVE); - } - - static boolean isAvailable() { - return Methods.REFLECTION_FACTORY != null - && Methods.WRITE_OBJECT != null - && Methods.READ_OBJECT != null - && Methods.READ_OBJECT_NO_DATA != null - && Methods.WRITE_REPLACE != null - && Methods.READ_RESOLVE != null; + Method method = getMethod(type, Methods.READ_RESOLVE); + return method == null ? directMethods(type).readResolve : method; } } diff --git a/java/fory-core/src/main/java25/org/apache/fory/memory/LittleEndian.java b/java/fory-core/src/main/java25/org/apache/fory/memory/LittleEndian.java index 3687b8ec59..7d6e524826 100644 --- a/java/fory-core/src/main/java25/org/apache/fory/memory/LittleEndian.java +++ b/java/fory-core/src/main/java25/org/apache/fory/memory/LittleEndian.java @@ -19,7 +19,14 @@ package org.apache.fory.memory; +import java.lang.invoke.MethodHandles; +import java.lang.invoke.VarHandle; +import java.nio.ByteOrder; + public class LittleEndian { + private static final VarHandle BYTE_ARRAY_LONG = + MethodHandles.byteArrayViewVarHandle(long[].class, ByteOrder.LITTLE_ENDIAN); + public static int putVarUint36Small(byte[] arr, int index, long v) { if (v >>> 7 == 0) { arr[index] = (byte) v; @@ -56,24 +63,10 @@ private static int bigWriteUint36(byte[] arr, int index, long v) { } public static long getInt64(byte[] o, int index) { - return ((long) o[index] & 0xff) - | (((long) o[index + 1] & 0xff) << 8) - | (((long) o[index + 2] & 0xff) << 16) - | (((long) o[index + 3] & 0xff) << 24) - | (((long) o[index + 4] & 0xff) << 32) - | (((long) o[index + 5] & 0xff) << 40) - | (((long) o[index + 6] & 0xff) << 48) - | (((long) o[index + 7] & 0xff) << 56); + return (long) BYTE_ARRAY_LONG.get(o, index); } public static void putInt64(byte[] o, int index, long value) { - o[index] = (byte) value; - o[index + 1] = (byte) (value >>> 8); - o[index + 2] = (byte) (value >>> 16); - o[index + 3] = (byte) (value >>> 24); - o[index + 4] = (byte) (value >>> 32); - o[index + 5] = (byte) (value >>> 40); - o[index + 6] = (byte) (value >>> 48); - o[index + 7] = (byte) (value >>> 56); + BYTE_ARRAY_LONG.set(o, index, value); } } diff --git a/java/fory-core/src/main/java25/org/apache/fory/serializer/PlatformStringUtils.java b/java/fory-core/src/main/java25/org/apache/fory/serializer/PlatformStringUtils.java index 4eef97442b..81d55e98c6 100644 --- a/java/fory-core/src/main/java25/org/apache/fory/serializer/PlatformStringUtils.java +++ b/java/fory-core/src/main/java25/org/apache/fory/serializer/PlatformStringUtils.java @@ -140,43 +140,21 @@ private static StringHandles noAccess() { } static Object getStringValue(String value) { - checkStringAccess("String.value"); return STRING_VALUE_HANDLE.get(value); } static byte getStringCoder(String value) { - checkStringAccess("String.coder"); - if (STRING_CODER_HANDLE == null) { - throw new UnsupportedOperationException("String.coder is not available on this JDK"); - } return (byte) STRING_CODER_HANDLE.get(value); } static int getStringOffset(String value) { - checkStringAccess("String.offset"); - if (STRING_OFFSET_HANDLE == null) { - throw new UnsupportedOperationException("String.offset is not available on this JDK"); - } return (int) STRING_OFFSET_HANDLE.get(value); } static int getStringCount(String value) { - checkStringAccess("String.count"); - if (STRING_COUNT_HANDLE == null) { - throw new UnsupportedOperationException("String.count is not available on this JDK"); - } return (int) STRING_COUNT_HANDLE.get(value); } - private static void checkStringAccess(String target) { - if (!JDK_STRING_FIELD_ACCESS) { - throw new UnsupportedOperationException( - target - + " private access is unavailable; open java.base/java.lang.invoke to " - + "org.apache.fory.core"); - } - } - static long getCharsLong(char[] chars, int charIndex) { long c0 = chars[charIndex]; long c1 = chars[charIndex + 1]; diff --git a/java/fory-core/src/main/java25/org/apache/fory/serializer/SerializationHookLookup.java b/java/fory-core/src/main/java25/org/apache/fory/serializer/SerializationHookLookup.java deleted file mode 100644 index bd756794f8..0000000000 --- a/java/fory-core/src/main/java25/org/apache/fory/serializer/SerializationHookLookup.java +++ /dev/null @@ -1,144 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.fory.serializer; - -import java.io.ObjectInputStream; -import java.io.ObjectOutputStream; -import java.lang.invoke.MethodHandle; -import java.lang.invoke.MethodHandles; -import java.lang.invoke.SerializedLambda; -import java.lang.reflect.Method; -import java.lang.reflect.Modifier; -import org.apache.fory.collection.ClassValueCache; -import org.apache.fory.platform.internal._JDKAccess; -import org.apache.fory.util.ExceptionUtils; - -final class SerializationHookLookup { - private SerializationHookLookup() {} - - static MethodHandle readResolveHandle(Class type, Method method) - throws IllegalAccessException { - try { - return _JDKAccess._trustedLookup(type).unreflect(method); - } catch (IllegalArgumentException e) { - if (type != SerializedLambda.class) { - throw e; - } - // JDK25 rejects SerializedLambda itself as a privateLookupIn target. Reflective access still - // honors the same java.base/java.lang.invoke open requirement and preserves the serializer - // path used by JDK8-24. - try { - method.setAccessible(true); - } catch (RuntimeException inaccessible) { - throw new IllegalStateException( - "SerializedLambda readResolve requires java.base/java.lang.invoke to be open to " - + "org.apache.fory.core", - inaccessible); - } - return MethodHandles.lookup().unreflect(method); - } - } - - private static final ClassValueCache methodsCache = ClassValueCache.newClassKeyCache(32); - - private static final class Methods { - private final Method writeObject; - private final Method readObject; - private final Method readObjectNoData; - private final Method writeReplace; - private final Method readResolve; - - private Methods(Class type) { - writeObject = getPrivateMethod(type, "writeObject", void.class, ObjectOutputStream.class); - readObject = getPrivateMethod(type, "readObject", void.class, ObjectInputStream.class); - readObjectNoData = getPrivateMethod(type, "readObjectNoData", void.class); - writeReplace = getInheritableObjectMethod(type, "writeReplace"); - readResolve = getInheritableObjectMethod(type, "readResolve"); - } - } - - private static Methods methods(Class type) { - return methodsCache.get(type, () -> new Methods(type)); - } - - private static Method getPrivateMethod( - Class type, String methodName, Class returnType, Class... parameterTypes) { - try { - Method method = type.getDeclaredMethod(methodName, parameterTypes); - int modifiers = method.getModifiers(); - if (Modifier.isPrivate(modifiers) - && !Modifier.isStatic(modifiers) - && method.getReturnType() == returnType) { - return method; - } - } catch (NoSuchMethodException | SecurityException e) { - ExceptionUtils.ignore(e); - } - return null; - } - - private static Method getInheritableObjectMethod(Class type, String methodName) { - Class cls = type; - while (cls != null) { - try { - Method method = cls.getDeclaredMethod(methodName); - int modifiers = method.getModifiers(); - if (!Modifier.isStatic(modifiers) && method.getReturnType() == Object.class) { - return method; - } - return null; - } catch (NoSuchMethodException e) { - ExceptionUtils.ignore(e); - } catch (SecurityException e) { - return null; - } - cls = cls.getSuperclass(); - } - return null; - } - - static Method getWriteObjectMethod(Class type) { - return methods(type).writeObject; - } - - static Method getReadObjectMethod(Class type) { - return methods(type).readObject; - } - - static Method getReadObjectNoDataMethod(Class type) { - return methods(type).readObjectNoData; - } - - static MethodHandle getDefaultReadObjectHandle(Class type) { - return null; - } - - static Method getWriteReplaceMethod(Class type) { - return methods(type).writeReplace; - } - - static Method getReadResolveMethod(Class type) { - return methods(type).readResolve; - } - - static boolean isAvailable() { - return true; - } -} From 4d5fe5cab63ad7402c1e11425650030a19ee98f6 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Tue, 2 Jun 2026 15:14:33 +0800 Subject: [PATCH 44/69] refactor(java): clean constructor bypass allocation --- .agents/languages/java.md | 5 +- .../jpms_tests/src/main/java/module-info.java | 18 +++--- .../apache/fory/integration_tests/Test.java | 14 ++--- .../constructor/PrivateConstructorBean.java | 45 ++++++++++++++ .../PublicSerializerValueSerializer.java | 5 +- .../JpmsFieldAccessorTest.java | 18 ++++-- java/fory-core/pom.xml | 20 +++---- ...r.java => ConstructorBypassAllocator.java} | 6 +- .../apache/fory/reflect/ObjectCreators.java | 59 ++++++++++++------- ...r.java => ConstructorBypassAllocator.java} | 54 +++++++++-------- .../fory-core/native-image.properties | 2 +- 11 files changed, 163 insertions(+), 83 deletions(-) create mode 100644 integration_tests/jpms_tests/src/main/java/org/apache/fory/integration_tests/constructor/PrivateConstructorBean.java rename java/fory-core/src/main/java/org/apache/fory/reflect/{UnsafeObjectAllocator.java => ConstructorBypassAllocator.java} (91%) rename java/fory-core/src/main/java25/org/apache/fory/reflect/{UnsafeObjectAllocator.java => ConstructorBypassAllocator.java} (65%) diff --git a/.agents/languages/java.md b/.agents/languages/java.md index 54e22012a0..105aa46f3b 100644 --- a/.agents/languages/java.md +++ b/.agents/languages/java.md @@ -59,9 +59,12 @@ Load this file when changing anything under `java/` or when Java drives a cross- - If changes touch GraalVM bootstrap, serializer retention, native-image metadata, or `ObjectStreamSerializer` GraalVM behavior, verify the native-image build and run the produced binary; a plain Java compile is insufficient. - Put latest-JDK or virtual-thread tests in the latest-JDK test modules with the matching compiler/profile floor, and centralize runtime-version probing in existing compatibility utilities. - For JDK25+ zero-Unsafe work, preserve serializer-family selection by type and configuration. Do not switch a type from `ObjectStreamSerializer` or another Fory serializer family to `JavaSerializer`, a JDK stream fallback, or any broad `java.* Serializable` fallback by JDK version or no-arg-constructor shape. +- JDK25+ zero-Unsafe runtime support is a JPMS named-module design. Fory core must run with `--add-opens=java.base/java.lang.invoke=org.apache.fory.core` so it can obtain the trusted lookup; missing this open is an invalid access configuration, not a reason to open per-package JDK internals or switch serializer/object-creation families. Do not use `ALL-UNNAMED` as the zero-Unsafe verification target. - Keep JDK25+ unsafe-removal implementation invariants in agent/design docs and tests, not user guides. User guides should document user actions such as `--sun-misc-unsafe-memory-access=deny`, `java.base/java.lang.invoke` opens, JDK26+ `--enable-final-field-mutation`, package opens for user named modules, and public APIs such as `@ForyConstructor` and `BaseFory.registerConstructor(...)`; do not expose internal serializer names, owner-model rationale, or avoided fallback strategies there. - For JDK25+ zero-Unsafe final-field behavior, distinguish JDK25 from JDK26+: JDK25 has no final-field mutation flag requirement, while JDK26+ requires `--enable-final-field-mutation` for post-construction final-field writes. -- For JDK25+ object creation, do not use `sun.reflect.ReflectionFactory` or `jdk.internal.reflect.ReflectionFactory`. Serializable classes without a no-arg constructor may use `ObjectStreamClass.newInstance` through the trusted lookup owner; non-Serializable classes without a no-arg constructor require `@ForyConstructor`, `BaseFory.registerConstructor(...)`, record construction, or a custom serializer. +- For JDK25+ object creation, do not use `sun.reflect.ReflectionFactory` or `jdk.internal.reflect.ReflectionFactory`. The shared `ObjectCreators` facade should route ObjectStream-compatible construction through `ParentNoArgCtrObjectCreator`; the replaceable constructor-bypass allocator owns the JDK8-24 Unsafe allocation path and the JDK25+ `ObjectStreamClass.newInstance` path. Serializable classes without a no-arg constructor may use that trusted-lookup owner; non-Serializable classes without a no-arg constructor require `@ForyConstructor`, `BaseFory.registerConstructor(...)`, record construction, or a custom serializer. +- In JDK25+ constructor-bypass allocation, cache `ObjectStreamClass.lookupAny(...)` per class and let `ObjectStreamClass.newInstance` validate unsupported classes. Do not add redundant `Serializable` prechecks or per-call descriptor lookups, and keep exception/message construction in cold helper methods. +- Keep the Java25 `_Lookup` and `DefineClass` overlays unless a future refactor can merge them without exposing Unsafe to the JDK25 class graph or replacing direct hidden-class APIs with reflective wrappers. Root `_Lookup` uses Unsafe for the JDK8-24 trusted-lookup fast path, while Java25 `_Lookup` uses the required `java.lang.invoke` open. Root `DefineClass` targets Java 8 bytecode and cannot directly reference `Lookup#defineHiddenClass` or `Lookup.ClassOption.NESTMATE`; Java25 `DefineClass` owns that direct API use. - Treat `ByteArrayOutputStream` and `ByteArrayInputStream` as ordinary streams on every JDK. Do not restore private-buffer wrapping for JDK8-24 performance, because that reintroduces `java.base/java.io` private-field ownership and module-open requirements. diff --git a/integration_tests/jpms_tests/src/main/java/module-info.java b/integration_tests/jpms_tests/src/main/java/module-info.java index c7e714742d..b769ae2bbc 100644 --- a/integration_tests/jpms_tests/src/main/java/module-info.java +++ b/integration_tests/jpms_tests/src/main/java/module-info.java @@ -18,15 +18,17 @@ */ module org.apache.fory.integration_tests { + requires org.apache.fory.core; + requires org.apache.fory.format; + requires org.apache.fory.test.core; - requires org.apache.fory.core; - requires org.apache.fory.format; - requires org.apache.fory.test.core; + // we can't really test any classes from this module because it only contains test-classes + requires org.apache.fory.test.suite; - // we can't really test any classes from this module because it only contains test-classes - requires org.apache.fory.test.suite; + exports org.apache.fory.integration_tests.constructor; + exports org.apache.fory.integration_tests.model; + exports org.apache.fory.integration_tests.publicserializer; - exports org.apache.fory.integration_tests.model; - exports org.apache.fory.integration_tests.publicserializer; - opens org.apache.fory.integration_tests.model to org.apache.fory.core; + opens org.apache.fory.integration_tests.model to + org.apache.fory.core; } diff --git a/integration_tests/jpms_tests/src/main/java/org/apache/fory/integration_tests/Test.java b/integration_tests/jpms_tests/src/main/java/org/apache/fory/integration_tests/Test.java index 2e3d778034..c1a9d58bea 100644 --- a/integration_tests/jpms_tests/src/main/java/org/apache/fory/integration_tests/Test.java +++ b/integration_tests/jpms_tests/src/main/java/org/apache/fory/integration_tests/Test.java @@ -24,15 +24,15 @@ import org.apache.fory.test.bean.Foo; /** - * A test class that simply references classes from the various Fory artifacts to check whether - * they specify the module names referenced in the module-info descriptor. + * A test class that simply references classes from the various Fory artifacts to check whether they + * specify the module names referenced in the module-info descriptor. */ public class Test { - public static void main(String[] args) { - final Fory fory = Fory.builder().build(); - fory.serialize(Foo.create()); + public static void main(String[] args) { + final Fory fory = Fory.builder().build(); + fory.serialize(Foo.create()); - Encoders.bean(Foo.class, fory); - } + Encoders.bean(Foo.class, fory); + } } diff --git a/integration_tests/jpms_tests/src/main/java/org/apache/fory/integration_tests/constructor/PrivateConstructorBean.java b/integration_tests/jpms_tests/src/main/java/org/apache/fory/integration_tests/constructor/PrivateConstructorBean.java new file mode 100644 index 0000000000..55187043b9 --- /dev/null +++ b/integration_tests/jpms_tests/src/main/java/org/apache/fory/integration_tests/constructor/PrivateConstructorBean.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.integration_tests.constructor; + +import org.apache.fory.annotation.ForyConstructor; + +public final class PrivateConstructorBean { + private final String name; + private final int age; + + @ForyConstructor({"name", "age"}) + private PrivateConstructorBean(String name, int age) { + this.name = name; + this.age = age; + } + + public static PrivateConstructorBean of(String name, int age) { + return new PrivateConstructorBean(name, age); + } + + public String name() { + return name; + } + + public int age() { + return age; + } +} diff --git a/integration_tests/jpms_tests/src/main/java/org/apache/fory/integration_tests/publicserializer/PublicSerializerValueSerializer.java b/integration_tests/jpms_tests/src/main/java/org/apache/fory/integration_tests/publicserializer/PublicSerializerValueSerializer.java index 25b2aa2e43..bf37b26eed 100644 --- a/integration_tests/jpms_tests/src/main/java/org/apache/fory/integration_tests/publicserializer/PublicSerializerValueSerializer.java +++ b/integration_tests/jpms_tests/src/main/java/org/apache/fory/integration_tests/publicserializer/PublicSerializerValueSerializer.java @@ -21,11 +21,12 @@ import org.apache.fory.context.ReadContext; import org.apache.fory.context.WriteContext; -import org.apache.fory.serializer.Serializer; import org.apache.fory.resolver.TypeResolver; +import org.apache.fory.serializer.Serializer; public final class PublicSerializerValueSerializer extends Serializer { - public PublicSerializerValueSerializer(TypeResolver typeResolver, Class type) { + public PublicSerializerValueSerializer( + TypeResolver typeResolver, Class type) { super(typeResolver.getConfig(), type); } diff --git a/integration_tests/jpms_tests/src/test/java/org/apache/fory/integration_tests/JpmsFieldAccessorTest.java b/integration_tests/jpms_tests/src/test/java/org/apache/fory/integration_tests/JpmsFieldAccessorTest.java index 5eea91bcbb..c1908ec481 100644 --- a/integration_tests/jpms_tests/src/test/java/org/apache/fory/integration_tests/JpmsFieldAccessorTest.java +++ b/integration_tests/jpms_tests/src/test/java/org/apache/fory/integration_tests/JpmsFieldAccessorTest.java @@ -21,6 +21,7 @@ import java.lang.reflect.Field; import org.apache.fory.Fory; +import org.apache.fory.integration_tests.constructor.PrivateConstructorBean; import org.apache.fory.integration_tests.model.PrivateFieldBean; import org.apache.fory.integration_tests.publicserializer.PublicSerializerValue; import org.apache.fory.integration_tests.publicserializer.PublicSerializerValueSerializer; @@ -42,16 +43,23 @@ public void testPrivateFieldAccess() throws Exception { @Test public void testPrivateFinalFieldSerialization() { Fory fory = - Fory.builder() - .withXlang(false) - .withCodegen(false) - .requireClassRegistration(false) - .build(); + Fory.builder().withXlang(false).withCodegen(false).requireClassRegistration(false).build(); PrivateFieldBean result = (PrivateFieldBean) fory.deserialize(fory.serialize(new PrivateFieldBean(13))); Assert.assertEquals(result.value(), 13); } + @Test + public void testPrivateConstructorBinding() { + Fory fory = + Fory.builder().withXlang(false).withCodegen(false).requireClassRegistration(false).build(); + PrivateConstructorBean result = + (PrivateConstructorBean) + fory.deserialize(fory.serialize(PrivateConstructorBean.of("Ada", 37))); + Assert.assertEquals(result.name(), "Ada"); + Assert.assertEquals(result.age(), 37); + } + @Test public void testPublicSerializerInExportedPackage() { Fory fory = Fory.builder().withXlang(false).requireClassRegistration(false).build(); diff --git a/java/fory-core/pom.xml b/java/fory-core/pom.xml index bf8ccb47f0..a738f3fda5 100644 --- a/java/fory-core/pom.xml +++ b/java/fory-core/pom.xml @@ -444,7 +444,7 @@ - + @@ -477,8 +477,8 @@ file="${jdk25.mrjar.check.dir}/META-INF/versions/25/org/apache/fory/reflect/HiddenFieldAccessorFactory.class" property="jdk25.hiddenfieldaccessorfactory.present"/> + file="${jdk25.mrjar.check.dir}/META-INF/versions/25/org/apache/fory/reflect/ConstructorBypassAllocator.class" + property="jdk25.constructorbypassallocator.present"/> @@ -513,8 +513,8 @@ unless="jdk25.hiddenfieldaccessorfactory.present" message="JDK25 multi-release HiddenFieldAccessorFactory class is missing from the packaged fory-core jar."/> + unless="jdk25.constructorbypassallocator.present" + message="JDK25 multi-release ConstructorBypassAllocator class is missing from the packaged fory-core jar."/> @@ -559,7 +559,7 @@ + name="META-INF/versions/25/org/apache/fory/reflect/ConstructorBypassAllocator.java"/> @@ -576,8 +576,8 @@ file="${jdk25.sources.check.dir}/META-INF/versions/25/org/apache/fory/reflect/HiddenFieldAccessorFactory.java" property="jdk25.hiddenfieldaccessorfactory.source.present"/> + file="${jdk25.sources.check.dir}/META-INF/versions/25/org/apache/fory/reflect/ConstructorBypassAllocator.java" + property="jdk25.constructorbypassallocator.source.present"/> @@ -594,8 +594,8 @@ unless="jdk25.hiddenfieldaccessorfactory.source.present" message="JDK25 versioned HiddenFieldAccessorFactory source is missing from the source jar."/> + unless="jdk25.constructorbypassallocator.source.present" + message="JDK25 versioned ConstructorBypassAllocator source is missing from the source jar."/> diff --git a/java/fory-core/src/main/java/org/apache/fory/reflect/UnsafeObjectAllocator.java b/java/fory-core/src/main/java/org/apache/fory/reflect/ConstructorBypassAllocator.java similarity index 91% rename from java/fory-core/src/main/java/org/apache/fory/reflect/UnsafeObjectAllocator.java rename to java/fory-core/src/main/java/org/apache/fory/reflect/ConstructorBypassAllocator.java index f1e14b79b0..4124229637 100644 --- a/java/fory-core/src/main/java/org/apache/fory/reflect/UnsafeObjectAllocator.java +++ b/java/fory-core/src/main/java/org/apache/fory/reflect/ConstructorBypassAllocator.java @@ -26,12 +26,12 @@ import org.apache.fory.platform.internal._UnsafeUtils; import sun.misc.Unsafe; -/** Internal JDK8-24 allocator used by object creators. */ +/** Internal JDK8-24 constructor-bypass allocator used by object creators. */ @Internal -final class UnsafeObjectAllocator { +final class ConstructorBypassAllocator { private static final Unsafe UNSAFE = AndroidSupport.IS_ANDROID ? null : _UnsafeUtils.UNSAFE; - private UnsafeObjectAllocator() {} + private ConstructorBypassAllocator() {} static T allocate(Class type) { if (UNSAFE == null || JdkVersion.MAJOR_VERSION >= 25) { diff --git a/java/fory-core/src/main/java/org/apache/fory/reflect/ObjectCreators.java b/java/fory-core/src/main/java/org/apache/fory/reflect/ObjectCreators.java index dd361c1c86..588d62b09d 100644 --- a/java/fory-core/src/main/java/org/apache/fory/reflect/ObjectCreators.java +++ b/java/fory-core/src/main/java/org/apache/fory/reflect/ObjectCreators.java @@ -133,11 +133,11 @@ static ObjectCreator createObjectCreator( if (noArgConstructor != null) { return new DeclaredNoArgCtrObjectCreator<>(type); } else { - return new UnsafeObjectCreator<>(type); + return new ConstructorBypassObjectCreator<>(type); } } if (noArgConstructor == null) { - return new UnsafeObjectCreator<>(type); + return new ConstructorBypassObjectCreator<>(type); } return new DeclaredNoArgCtrObjectCreator<>(type); } @@ -151,8 +151,8 @@ private static ObjectCreator createObjectStreamCreator(Class type) { return new UnsupportedObjectCreator<>( type, "Android cannot create " + type + " without an accessible no-arg constructor"); } - if (GraalvmSupport.IN_GRAALVM_NATIVE_IMAGE || JdkVersion.MAJOR_VERSION >= 25) { - return new UnsafeObjectCreator<>(type); + if (GraalvmSupport.IN_GRAALVM_NATIVE_IMAGE) { + return new ConstructorBypassObjectCreator<>(type); } return new ParentNoArgCtrObjectCreator<>(type); } @@ -343,6 +343,10 @@ private static MethodHandle constructorHandle(Class type, Constructor cons } } + private static ForyException makeException(Class type, Throwable cause) { + return new ForyException("Failed to create instance for " + type, cause); + } + private static final class ReflectiveNoArgCtrObjectCreator extends ObjectCreator { private final Constructor constructor; @@ -361,7 +365,7 @@ public T newInstance() { try { return constructor.newInstance(); } catch (Exception e) { - throw new ForyException("Failed to create instance using no-arg constructor: " + type, e); + throw makeException(type, e); } } @@ -406,10 +410,8 @@ private ConstructorObjectCreator(Class type, ConstructorMatch match) { declaringClasses = match.declaringClasses; fieldTypes = match.fieldTypes; finalFields = match.finalFields; - try { - constructor.setAccessible(true); - } catch (RuntimeException e) { - throw new ForyException("Failed to make constructor accessible for " + type, e); + if (handle == null) { + makeConstructorAccessible(type, constructor); } } @@ -458,10 +460,17 @@ public boolean isOnlyPublicConstructor() { return true; } + private static void makeConstructorAccessible(Class type, Constructor constructor) { + try { + constructor.setAccessible(true); + } catch (RuntimeException e) { + throw new ForyException("Failed to make constructor accessible for " + type, e); + } + } + @Override public T newInstance() { - throw new ForyException( - "JDK25 zero-Unsafe mode requires constructor field values to create " + type); + throw constructorArgsRequired(type); } @Override @@ -472,20 +481,25 @@ public T newInstanceWithArguments(Object... arguments) { } return (T) handle.invoke(arguments); } catch (Throwable e) { - throw new ForyException("Failed to create instance using constructor: " + type, e); + throw makeException(type, e); } } + + private static ForyException constructorArgsRequired(Class type) { + return new ForyException( + "JDK25 zero-Unsafe mode requires constructor field values to create " + type); + } } - private static final class UnsafeObjectCreator extends ObjectCreator { + private static final class ConstructorBypassObjectCreator extends ObjectCreator { - public UnsafeObjectCreator(Class type) { + public ConstructorBypassObjectCreator(Class type) { super(type); } @Override public T newInstance() { - return UnsafeObjectAllocator.allocate(type); + return ConstructorBypassAllocator.allocate(type); } @Override @@ -507,7 +521,7 @@ public T newInstance() { try { return (T) handle.invoke(); } catch (Throwable e) { - throw new RuntimeException(e); + throw makeException(type, e); } } @@ -558,7 +572,7 @@ public T newInstanceWithArguments(Object... arguments) { return (T) handle.invoke(arguments); } } catch (Throwable e) { - throw new ForyException("Failed to create record instance: " + type, e); + throw makeException(type, e); } } } @@ -572,9 +586,8 @@ public static final class ParentNoArgCtrObjectCreator extends ObjectCreator type) { super(type); if (JdkVersion.MAJOR_VERSION >= 25) { - throw new ForyException( - "ReflectionFactory object creation is unavailable in JDK25+ zero-Unsafe mode for " - + type); + constructor = null; + return; } this.constructor = createSerializationConstructor(type); } @@ -634,11 +647,13 @@ private static boolean validSerializationConstructor( @Override public T newInstance() { + if (constructor == null) { + return ConstructorBypassAllocator.allocate(type); + } try { return constructor.newInstance(); } catch (Exception e) { - throw new ForyException( - "Failed to create instance, please provide a no-arg constructor for " + type, e); + throw makeException(type, e); } } diff --git a/java/fory-core/src/main/java25/org/apache/fory/reflect/UnsafeObjectAllocator.java b/java/fory-core/src/main/java25/org/apache/fory/reflect/ConstructorBypassAllocator.java similarity index 65% rename from java/fory-core/src/main/java25/org/apache/fory/reflect/UnsafeObjectAllocator.java rename to java/fory-core/src/main/java25/org/apache/fory/reflect/ConstructorBypassAllocator.java index 35809db017..932a3aab85 100644 --- a/java/fory-core/src/main/java25/org/apache/fory/reflect/UnsafeObjectAllocator.java +++ b/java/fory-core/src/main/java25/org/apache/fory/reflect/ConstructorBypassAllocator.java @@ -20,35 +20,31 @@ package org.apache.fory.reflect; import java.io.ObjectStreamClass; -import java.io.Serializable; import java.lang.invoke.MethodHandle; import java.lang.invoke.MethodType; -import java.lang.reflect.InvocationTargetException; import org.apache.fory.annotation.Internal; +import org.apache.fory.collection.ClassValueCache; import org.apache.fory.exception.ForyException; import org.apache.fory.platform.internal._JDKAccess; -import org.apache.fory.util.ExceptionUtils; -/** JDK25 replacement for the JDK8-24 Unsafe allocator. */ +/** JDK25 replacement for the JDK8-24 constructor-bypass allocator. */ @Internal -final class UnsafeObjectAllocator { - private UnsafeObjectAllocator() {} +final class ConstructorBypassAllocator { + private ConstructorBypassAllocator() {} static T allocate(Class type) { - if (Serializable.class.isAssignableFrom(type)) { - try { - return type.cast(ObjectStreamClassAccess.newInstance(type)); - } catch (UnsupportedOperationException e) { - throw unsupported(type, e); - } catch (InstantiationException e) { - throw unsupported(type, e); - } catch (InvocationTargetException e) { - throw ExceptionUtils.throwException(e.getTargetException()); - } catch (Throwable e) { - throw new ForyException("Failed to create an instance for " + type, e); - } + try { + return type.cast(ObjectStreamClassAccess.newInstance(type)); + } catch (Throwable e) { + throw handleAllocationException(type, e); + } + } + + private static RuntimeException handleAllocationException(Class type, Throwable cause) { + if (cause instanceof UnsupportedOperationException || cause instanceof InstantiationException) { + return unsupported(type, cause); } - throw unsupported(type, null); + return new ForyException("Failed to create an instance for " + type, cause); } private static ForyException unsupported(Class type, Throwable cause) { @@ -63,6 +59,8 @@ private static ForyException unsupported(Class type, Throwable cause) { } private static final class ObjectStreamClassAccess { + private static final ClassValueCache CLASSES = + ClassValueCache.newClassKeyCache(32); private static final MethodHandle NEW_INSTANCE; private static final Throwable INIT_ERROR; @@ -85,12 +83,20 @@ private static final class ObjectStreamClassAccess { private static Object newInstance(Class type) throws Throwable { MethodHandle handle = NEW_INSTANCE; if (handle == null) { - throw new ForyException( - "JDK25+ Serializable object creation requires java.base/java.lang.invoke to be open " - + "to org.apache.fory.core", - INIT_ERROR); + throw missingLookup(); } - return handle.invoke(ObjectStreamClass.lookupAny(type)); + return handle.invoke(objectStreamClass(type)); + } + + private static ObjectStreamClass objectStreamClass(Class type) { + return CLASSES.get(type, () -> ObjectStreamClass.lookupAny(type)); + } + + private static ForyException missingLookup() { + return new ForyException( + "JDK25+ Serializable object creation requires java.base/java.lang.invoke to be open " + + "to org.apache.fory.core", + INIT_ERROR); } } } diff --git a/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties b/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties index 69fb5bfd0e..4a135008bb 100644 --- a/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties +++ b/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties @@ -256,7 +256,7 @@ Args=--initialize-at-build-time=org.apache.fory.annotation.ForyField$Dynamic,\ org.apache.fory.reflect.TypeRef,\ org.apache.fory.reflect.ObjectCreators,\ org.apache.fory.reflect.ObjectCreators$ConstructorObjectCreator,\ - org.apache.fory.reflect.ObjectCreators$UnsafeObjectCreator,\ + org.apache.fory.reflect.ObjectCreators$ConstructorBypassObjectCreator,\ org.apache.fory.reflect.ObjectCreators$DeclaredNoArgCtrObjectCreator,\ org.apache.fory.reflect.ObjectCreators$ParentNoArgCtrObjectCreator,\ org.apache.fory.reflect.ObjectCreators$RecordObjectCreator,\ From 7b2b42b45dc699226ccf1a587630a3d26ead1fc5 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Tue, 2 Jun 2026 15:22:12 +0800 Subject: [PATCH 45/69] refactor(java): make constructor bypass allocator typed --- .agents/languages/java.md | 2 +- .../reflect/ConstructorBypassAllocator.java | 28 ++++++++++++++----- .../apache/fory/reflect/ObjectCreators.java | 12 ++++++-- .../reflect/ConstructorBypassAllocator.java | 25 ++++++++--------- 4 files changed, 43 insertions(+), 24 deletions(-) diff --git a/.agents/languages/java.md b/.agents/languages/java.md index 105aa46f3b..46f60fab70 100644 --- a/.agents/languages/java.md +++ b/.agents/languages/java.md @@ -63,7 +63,7 @@ Load this file when changing anything under `java/` or when Java drives a cross- - Keep JDK25+ unsafe-removal implementation invariants in agent/design docs and tests, not user guides. User guides should document user actions such as `--sun-misc-unsafe-memory-access=deny`, `java.base/java.lang.invoke` opens, JDK26+ `--enable-final-field-mutation`, package opens for user named modules, and public APIs such as `@ForyConstructor` and `BaseFory.registerConstructor(...)`; do not expose internal serializer names, owner-model rationale, or avoided fallback strategies there. - For JDK25+ zero-Unsafe final-field behavior, distinguish JDK25 from JDK26+: JDK25 has no final-field mutation flag requirement, while JDK26+ requires `--enable-final-field-mutation` for post-construction final-field writes. - For JDK25+ object creation, do not use `sun.reflect.ReflectionFactory` or `jdk.internal.reflect.ReflectionFactory`. The shared `ObjectCreators` facade should route ObjectStream-compatible construction through `ParentNoArgCtrObjectCreator`; the replaceable constructor-bypass allocator owns the JDK8-24 Unsafe allocation path and the JDK25+ `ObjectStreamClass.newInstance` path. Serializable classes without a no-arg constructor may use that trusted-lookup owner; non-Serializable classes without a no-arg constructor require `@ForyConstructor`, `BaseFory.registerConstructor(...)`, record construction, or a custom serializer. -- In JDK25+ constructor-bypass allocation, cache `ObjectStreamClass.lookupAny(...)` per class and let `ObjectStreamClass.newInstance` validate unsupported classes. Do not add redundant `Serializable` prechecks or per-call descriptor lookups, and keep exception/message construction in cold helper methods. +- In JDK25+ constructor-bypass allocation, the allocator is per type: pass the target class to the allocator constructor, cache `ObjectStreamClass.lookupAny(...)` in that instance, and expose an instance `allocate()` method. Let `ObjectStreamClass.newInstance` validate unsupported classes. Do not add redundant `Serializable` prechecks or per-call descriptor lookups, and keep exception/message construction in cold helper methods. - Keep the Java25 `_Lookup` and `DefineClass` overlays unless a future refactor can merge them without exposing Unsafe to the JDK25 class graph or replacing direct hidden-class APIs with reflective wrappers. Root `_Lookup` uses Unsafe for the JDK8-24 trusted-lookup fast path, while Java25 `_Lookup` uses the required `java.lang.invoke` open. Root `DefineClass` targets Java 8 bytecode and cannot directly reference `Lookup#defineHiddenClass` or `Lookup.ClassOption.NESTMATE`; Java25 `DefineClass` owns that direct API use. - Treat `ByteArrayOutputStream` and `ByteArrayInputStream` as ordinary streams on every JDK. Do not restore private-buffer wrapping for JDK8-24 performance, because that reintroduces diff --git a/java/fory-core/src/main/java/org/apache/fory/reflect/ConstructorBypassAllocator.java b/java/fory-core/src/main/java/org/apache/fory/reflect/ConstructorBypassAllocator.java index 4124229637..9cb859c63d 100644 --- a/java/fory-core/src/main/java/org/apache/fory/reflect/ConstructorBypassAllocator.java +++ b/java/fory-core/src/main/java/org/apache/fory/reflect/ConstructorBypassAllocator.java @@ -28,20 +28,34 @@ /** Internal JDK8-24 constructor-bypass allocator used by object creators. */ @Internal -final class ConstructorBypassAllocator { +final class ConstructorBypassAllocator { private static final Unsafe UNSAFE = AndroidSupport.IS_ANDROID ? null : _UnsafeUtils.UNSAFE; + private static final boolean UNSAFE_ALLOCATION_AVAILABLE = + UNSAFE != null && JdkVersion.MAJOR_VERSION < 25; - private ConstructorBypassAllocator() {} + private final Class type; - static T allocate(Class type) { - if (UNSAFE == null || JdkVersion.MAJOR_VERSION >= 25) { - throw new ForyException( - "Constructor-bypassing Unsafe allocation is unsupported in this runtime for " + type); + ConstructorBypassAllocator(Class type) { + this.type = type; + } + + T allocate() { + if (!UNSAFE_ALLOCATION_AVAILABLE) { + throw unsupported(type); } try { return (T) UNSAFE.allocateInstance(type); } catch (InstantiationException e) { - throw new ForyException("Failed to allocate instance for " + type, e); + throw allocationFailed(type, e); } } + + private static ForyException unsupported(Class type) { + return new ForyException( + "Constructor-bypassing Unsafe allocation is unsupported in this runtime for " + type); + } + + private static ForyException allocationFailed(Class type, InstantiationException cause) { + return new ForyException("Failed to allocate instance for " + type, cause); + } } diff --git a/java/fory-core/src/main/java/org/apache/fory/reflect/ObjectCreators.java b/java/fory-core/src/main/java/org/apache/fory/reflect/ObjectCreators.java index 588d62b09d..9a4842a27f 100644 --- a/java/fory-core/src/main/java/org/apache/fory/reflect/ObjectCreators.java +++ b/java/fory-core/src/main/java/org/apache/fory/reflect/ObjectCreators.java @@ -492,14 +492,16 @@ private static ForyException constructorArgsRequired(Class type) { } private static final class ConstructorBypassObjectCreator extends ObjectCreator { + private final ConstructorBypassAllocator allocator; public ConstructorBypassObjectCreator(Class type) { super(type); + allocator = new ConstructorBypassAllocator<>(type); } @Override public T newInstance() { - return ConstructorBypassAllocator.allocate(type); + return allocator.allocate(); } @Override @@ -582,14 +584,17 @@ public static final class ParentNoArgCtrObjectCreator extends ObjectCreator constructor; + private final ConstructorBypassAllocator allocator; public ParentNoArgCtrObjectCreator(Class type) { super(type); if (JdkVersion.MAJOR_VERSION >= 25) { constructor = null; + allocator = new ConstructorBypassAllocator<>(type); return; } this.constructor = createSerializationConstructor(type); + allocator = null; } private static Constructor createSerializationConstructor(Class type) { @@ -647,8 +652,9 @@ private static boolean validSerializationConstructor( @Override public T newInstance() { - if (constructor == null) { - return ConstructorBypassAllocator.allocate(type); + ConstructorBypassAllocator constructorBypassAllocator = allocator; + if (constructorBypassAllocator != null) { + return constructorBypassAllocator.allocate(); } try { return constructor.newInstance(); diff --git a/java/fory-core/src/main/java25/org/apache/fory/reflect/ConstructorBypassAllocator.java b/java/fory-core/src/main/java25/org/apache/fory/reflect/ConstructorBypassAllocator.java index 932a3aab85..8fbd5a44c9 100644 --- a/java/fory-core/src/main/java25/org/apache/fory/reflect/ConstructorBypassAllocator.java +++ b/java/fory-core/src/main/java25/org/apache/fory/reflect/ConstructorBypassAllocator.java @@ -23,18 +23,23 @@ import java.lang.invoke.MethodHandle; import java.lang.invoke.MethodType; import org.apache.fory.annotation.Internal; -import org.apache.fory.collection.ClassValueCache; import org.apache.fory.exception.ForyException; import org.apache.fory.platform.internal._JDKAccess; /** JDK25 replacement for the JDK8-24 constructor-bypass allocator. */ @Internal -final class ConstructorBypassAllocator { - private ConstructorBypassAllocator() {} +final class ConstructorBypassAllocator { + private final Class type; + private final ObjectStreamClass objectStreamClass; - static T allocate(Class type) { + ConstructorBypassAllocator(Class type) { + this.type = type; + objectStreamClass = ObjectStreamClass.lookupAny(type); + } + + T allocate() { try { - return type.cast(ObjectStreamClassAccess.newInstance(type)); + return type.cast(ObjectStreamClassAccess.newInstance(objectStreamClass)); } catch (Throwable e) { throw handleAllocationException(type, e); } @@ -59,8 +64,6 @@ private static ForyException unsupported(Class type, Throwable cause) { } private static final class ObjectStreamClassAccess { - private static final ClassValueCache CLASSES = - ClassValueCache.newClassKeyCache(32); private static final MethodHandle NEW_INSTANCE; private static final Throwable INIT_ERROR; @@ -80,16 +83,12 @@ private static final class ObjectStreamClassAccess { INIT_ERROR = error; } - private static Object newInstance(Class type) throws Throwable { + private static Object newInstance(ObjectStreamClass objectStreamClass) throws Throwable { MethodHandle handle = NEW_INSTANCE; if (handle == null) { throw missingLookup(); } - return handle.invoke(objectStreamClass(type)); - } - - private static ObjectStreamClass objectStreamClass(Class type) { - return CLASSES.get(type, () -> ObjectStreamClass.lookupAny(type)); + return handle.invoke(objectStreamClass); } private static ForyException missingLookup() { From b4c5947791b6901519fb240bdbcd594db4ba5c67 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Tue, 2 Jun 2026 15:38:29 +0800 Subject: [PATCH 46/69] refactor(java): close jdk25 unsafe class graph --- .agents/languages/java.md | 9 + benchmarks/java/pom.xml | 34 ++++ .../fory/benchmark/Jdk25MrJarCheck.java | 34 +++- .../{Benchmark.java => JmhBenchmarkMain.java} | 2 +- java/fory-core/pom.xml | 39 +++++ .../org/apache/fory/memory/MemoryBuffer.java | 24 +-- .../fory/reflect/FieldAccessorStrategy.java | 157 ++++++++++-------- .../fory/serializer/PlatformStringUtils.java | 16 +- .../fory/platform/internal/_UnsafeUtils.java | 27 +++ ...st.java => Jdk25UnsafeClassGraphTest.java} | 22 +-- 10 files changed, 255 insertions(+), 109 deletions(-) rename benchmarks/java/src/main/java/org/apache/fory/benchmark/{Benchmark.java => JmhBenchmarkMain.java} (97%) create mode 100644 java/fory-core/src/main/java25/org/apache/fory/platform/internal/_UnsafeUtils.java rename java/fory-core/src/test/java/org/apache/fory/builder/{BuilderUnsafeClassGraphTest.java => Jdk25UnsafeClassGraphTest.java} (80%) diff --git a/.agents/languages/java.md b/.agents/languages/java.md index 46f60fab70..ad37d7562a 100644 --- a/.agents/languages/java.md +++ b/.agents/languages/java.md @@ -79,6 +79,15 @@ Load this file when changing anything under `java/` or when Java drives a cross- - Root codegen and builder classes that still need Unsafe on JDK8-24 must route symbolic Unsafe access through a helper with a Java 25 replacement. Do not leave `_JDKAccess.unsafe()` or `sun.misc.Unsafe` references in JDK25-visible classes outside matching `java25` replacements. +- `_UnsafeUtils` is the JDK8-24 Unsafe owner only. It must have a Java25 replacement with no + `sun.misc.Unsafe` fields, methods, constructors, imports, or descriptors, and MR-JAR plus + benchmark checks must require that replacement so JDK25+ never links `jdk.unsupported` through + this utility. +- Multi-release replacement is class-file exact. Replacing `Foo.class` does not replace + `Foo$Bar.class`; root nested classes that contain Unsafe descriptors must either be eliminated, + moved under an exact replaceable owner, or have their direct Unsafe operations moved to the + already-overlaid top-level class. JDK25 package checks must run class-level `jdeps` over Fory + classes so nested leaks are caught without rejecting shaded third-party benchmark dependencies. - Java 25 multi-release classes never run on Android. Do not keep `AndroidSupport.IS_ANDROID` branches or imports under `src/main/java25`; Android compatibility belongs to the root sources. - String zero-copy construction is serializer-owned behavior. Keep private `String` constructor diff --git a/benchmarks/java/pom.xml b/benchmarks/java/pom.xml index a7bf23af8a..b04daf401e 100644 --- a/benchmarks/java/pom.xml +++ b/benchmarks/java/pom.xml @@ -265,6 +265,8 @@ name="META-INF/versions/25/org/apache/fory/memory/LittleEndian.class"/> + + @@ -306,6 +311,9 @@ + @@ -315,6 +323,32 @@ + + + + + + + + + + + + + + + + + jdkAccess = verifyRootClass("org.apache.fory.platform.internal._JDKAccess"); + Class unsafeUtils = verifyVersionedClass("org.apache.fory.platform.internal._UnsafeUtils"); verifyVersionedClass("org.apache.fory.reflect.FieldAccessorStrategy"); verifyVersionedClass("org.apache.fory.serializer.PlatformStringUtils"); - if (hasUnsafeField(jdkAccess)) { - throw new IllegalStateException("JDK25 benchmark jar loaded Unsafe-owning _JDKAccess"); - } + verifyNoUnsafeDescriptors(jdkAccess); + verifyNoUnsafeDescriptors(unsafeUtils); } private static void verifyMissing(String className) { @@ -78,12 +78,34 @@ private static Class verifyRootClass(String className) { return cls; } - private static boolean hasUnsafeField(Class jdkAccess) { - for (java.lang.reflect.Field field : jdkAccess.getDeclaredFields()) { - if (field.getName().equals("UNSAFE") || field.getType().getName().equals("sun.misc.Unsafe")) { + private static void verifyNoUnsafeDescriptors(Class cls) { + for (java.lang.reflect.Field field : cls.getDeclaredFields()) { + if (isUnsafeType(field.getType())) { + throw new IllegalStateException("JDK25 benchmark jar loaded Unsafe field in " + cls); + } + } + for (java.lang.reflect.Method method : cls.getDeclaredMethods()) { + if (isUnsafeType(method.getReturnType()) || hasUnsafeType(method.getParameterTypes())) { + throw new IllegalStateException("JDK25 benchmark jar loaded Unsafe method in " + cls); + } + } + for (java.lang.reflect.Constructor constructor : cls.getDeclaredConstructors()) { + if (hasUnsafeType(constructor.getParameterTypes())) { + throw new IllegalStateException("JDK25 benchmark jar loaded Unsafe constructor in " + cls); + } + } + } + + private static boolean hasUnsafeType(Class[] types) { + for (Class type : types) { + if (isUnsafeType(type)) { return true; } } return false; } + + private static boolean isUnsafeType(Class type) { + return type.getName().equals("sun.misc.Unsafe"); + } } diff --git a/benchmarks/java/src/main/java/org/apache/fory/benchmark/Benchmark.java b/benchmarks/java/src/main/java/org/apache/fory/benchmark/JmhBenchmarkMain.java similarity index 97% rename from benchmarks/java/src/main/java/org/apache/fory/benchmark/Benchmark.java rename to benchmarks/java/src/main/java/org/apache/fory/benchmark/JmhBenchmarkMain.java index d51f433eb9..27a69f087d 100644 --- a/benchmarks/java/src/main/java/org/apache/fory/benchmark/Benchmark.java +++ b/benchmarks/java/src/main/java/org/apache/fory/benchmark/JmhBenchmarkMain.java @@ -21,7 +21,7 @@ import org.openjdk.jmh.Main; -public class Benchmark { +public class JmhBenchmarkMain { // run from cmd: // cd .. && mvn -T10 install -DskipTests -Dcheckstyle.skip -Dlicense.skip -Dmaven.javadoc.skip // mvn exec:java -Dexec.args="-f 0 -wi 1 -i 1 -t 1 -w 1s -r 1s -rf csv" diff --git a/java/fory-core/pom.xml b/java/fory-core/pom.xml index a738f3fda5..b43f911ef0 100644 --- a/java/fory-core/pom.xml +++ b/java/fory-core/pom.xml @@ -440,6 +440,7 @@ + + @@ -503,6 +507,9 @@ + @@ -521,6 +528,30 @@ + + + + + + + + + + + + + + + + + @@ -554,6 +585,8 @@ + + @@ -587,6 +623,9 @@ + diff --git a/java/fory-core/src/main/java/org/apache/fory/memory/MemoryBuffer.java b/java/fory-core/src/main/java/org/apache/fory/memory/MemoryBuffer.java index 04733e0453..0b273e3976 100644 --- a/java/fory-core/src/main/java/org/apache/fory/memory/MemoryBuffer.java +++ b/java/fory-core/src/main/java/org/apache/fory/memory/MemoryBuffer.java @@ -106,20 +106,20 @@ public final class MemoryBuffer { /** Limits each raw Unsafe copy to let large copies hit safepoint polls between chunks. */ private static final long UNSAFE_COPY_THRESHOLD = 1024L * 1024L; + private static final long BUFFER_ADDRESS_FIELD_OFFSET = + AndroidSupport.IS_ANDROID ? -1 : bufferAddressFieldOffset(); + // Global allocator instance that can be customized private static volatile MemoryAllocator globalAllocator = new DefaultMemoryAllocator(); - private static final class DirectBufferAccess { - private static final long BUFFER_ADDRESS_FIELD_OFFSET; - - static { - try { - Field addressField = Buffer.class.getDeclaredField("address"); - BUFFER_ADDRESS_FIELD_OFFSET = UNSAFE.objectFieldOffset(addressField); - checkArgument(BUFFER_ADDRESS_FIELD_OFFSET != 0); - } catch (NoSuchFieldException e) { - throw new IllegalStateException(e); - } + private static long bufferAddressFieldOffset() { + try { + Field addressField = Buffer.class.getDeclaredField("address"); + long offset = UNSAFE.objectFieldOffset(addressField); + checkArgument(offset != 0); + return offset; + } catch (NoSuchFieldException e) { + throw new IllegalStateException(e); } } @@ -281,7 +281,7 @@ private static long getAddress(ByteBuffer buffer) { checkNotNull(buffer, "buffer is null"); checkArgument(buffer.isDirect(), "Can't get address of a non-direct ByteBuffer."); try { - return UNSAFE.getLong(buffer, DirectBufferAccess.BUFFER_ADDRESS_FIELD_OFFSET); + return UNSAFE.getLong(buffer, BUFFER_ADDRESS_FIELD_OFFSET); } catch (Throwable t) { throw new Error("Could not access direct byte buffer address field.", t); } diff --git a/java/fory-core/src/main/java/org/apache/fory/reflect/FieldAccessorStrategy.java b/java/fory-core/src/main/java/org/apache/fory/reflect/FieldAccessorStrategy.java index 23d4ebb535..4028919cfd 100644 --- a/java/fory-core/src/main/java/org/apache/fory/reflect/FieldAccessorStrategy.java +++ b/java/fory-core/src/main/java/org/apache/fory/reflect/FieldAccessorStrategy.java @@ -50,6 +50,91 @@ final class FieldAccessorStrategy { private FieldAccessorStrategy() {} + private static long fieldOffset(Field field) { + if (AndroidSupport.IS_ANDROID) { + return -1; + } + if (GraalvmSupport.isGraalBuildTime()) { + // Field offsets are rewritten by GraalVM and are not stable during native-image build time. + return -1; + } + return UNSAFE.objectFieldOffset(field); + } + + private static int accessKind(Field field) { + Class fieldType = field.getType(); + if (fieldType == boolean.class) { + return BOOLEAN_ACCESS; + } else if (fieldType == byte.class) { + return BYTE_ACCESS; + } else if (fieldType == char.class) { + return CHAR_ACCESS; + } else if (fieldType == short.class) { + return SHORT_ACCESS; + } else if (fieldType == int.class) { + return INT_ACCESS; + } else if (fieldType == long.class) { + return LONG_ACCESS; + } else if (fieldType == float.class) { + return FLOAT_ACCESS; + } else if (fieldType == double.class) { + return DOUBLE_ACCESS; + } + return OBJECT_ACCESS; + } + + private static void copyField( + long fieldOffset, + int accessKind, + Object sourceObject, + Object targetObject, + FieldAccessor accessor) { + switch (accessKind) { + case BOOLEAN_ACCESS: + UNSAFE.putBoolean(targetObject, fieldOffset, UNSAFE.getBoolean(sourceObject, fieldOffset)); + return; + case BYTE_ACCESS: + UNSAFE.putByte(targetObject, fieldOffset, UNSAFE.getByte(sourceObject, fieldOffset)); + return; + case CHAR_ACCESS: + UNSAFE.putChar(targetObject, fieldOffset, UNSAFE.getChar(sourceObject, fieldOffset)); + return; + case SHORT_ACCESS: + UNSAFE.putShort(targetObject, fieldOffset, UNSAFE.getShort(sourceObject, fieldOffset)); + return; + case INT_ACCESS: + UNSAFE.putInt(targetObject, fieldOffset, UNSAFE.getInt(sourceObject, fieldOffset)); + return; + case LONG_ACCESS: + UNSAFE.putLong(targetObject, fieldOffset, UNSAFE.getLong(sourceObject, fieldOffset)); + return; + case FLOAT_ACCESS: + UNSAFE.putFloat(targetObject, fieldOffset, UNSAFE.getFloat(sourceObject, fieldOffset)); + return; + case DOUBLE_ACCESS: + UNSAFE.putDouble(targetObject, fieldOffset, UNSAFE.getDouble(sourceObject, fieldOffset)); + return; + case OBJECT_ACCESS: + UNSAFE.putObject(targetObject, fieldOffset, UNSAFE.getObject(sourceObject, fieldOffset)); + return; + default: + accessor.putObject(targetObject, accessor.getObject(sourceObject)); + } + } + + private static void copyObjectField( + long fieldOffset, + int accessKind, + Object sourceObject, + Object targetObject, + FieldAccessor accessor) { + if (accessKind == OBJECT_ACCESS) { + UNSAFE.putObject(targetObject, fieldOffset, UNSAFE.getObject(sourceObject, fieldOffset)); + } else { + accessor.putObject(targetObject, accessor.getObject(sourceObject)); + } + } + private abstract static class UnsafeAccessor extends FieldAccessor { protected final long fieldOffset; private final int accessKind; @@ -60,82 +145,14 @@ private abstract static class UnsafeAccessor extends FieldAccessor { accessKind = accessKind(field); } - private static long fieldOffset(Field field) { - if (AndroidSupport.IS_ANDROID) { - return -1; - } - if (GraalvmSupport.isGraalBuildTime()) { - // Field offsets are rewritten by GraalVM and are not stable during native-image build time. - return -1; - } - return UNSAFE.objectFieldOffset(field); - } - - private static int accessKind(Field field) { - Class fieldType = field.getType(); - if (fieldType == boolean.class) { - return BOOLEAN_ACCESS; - } else if (fieldType == byte.class) { - return BYTE_ACCESS; - } else if (fieldType == char.class) { - return CHAR_ACCESS; - } else if (fieldType == short.class) { - return SHORT_ACCESS; - } else if (fieldType == int.class) { - return INT_ACCESS; - } else if (fieldType == long.class) { - return LONG_ACCESS; - } else if (fieldType == float.class) { - return FLOAT_ACCESS; - } else if (fieldType == double.class) { - return DOUBLE_ACCESS; - } - return OBJECT_ACCESS; - } - @Override public void copy(Object sourceObject, Object targetObject) { - switch (accessKind) { - case BOOLEAN_ACCESS: - UNSAFE.putBoolean( - targetObject, fieldOffset, UNSAFE.getBoolean(sourceObject, fieldOffset)); - return; - case BYTE_ACCESS: - UNSAFE.putByte(targetObject, fieldOffset, UNSAFE.getByte(sourceObject, fieldOffset)); - return; - case CHAR_ACCESS: - UNSAFE.putChar(targetObject, fieldOffset, UNSAFE.getChar(sourceObject, fieldOffset)); - return; - case SHORT_ACCESS: - UNSAFE.putShort(targetObject, fieldOffset, UNSAFE.getShort(sourceObject, fieldOffset)); - return; - case INT_ACCESS: - UNSAFE.putInt(targetObject, fieldOffset, UNSAFE.getInt(sourceObject, fieldOffset)); - return; - case LONG_ACCESS: - UNSAFE.putLong(targetObject, fieldOffset, UNSAFE.getLong(sourceObject, fieldOffset)); - return; - case FLOAT_ACCESS: - UNSAFE.putFloat(targetObject, fieldOffset, UNSAFE.getFloat(sourceObject, fieldOffset)); - return; - case DOUBLE_ACCESS: - UNSAFE.putDouble(targetObject, fieldOffset, UNSAFE.getDouble(sourceObject, fieldOffset)); - return; - case OBJECT_ACCESS: - UNSAFE.putObject(targetObject, fieldOffset, UNSAFE.getObject(sourceObject, fieldOffset)); - return; - default: - putObject(targetObject, getObject(sourceObject)); - } + copyField(fieldOffset, accessKind, sourceObject, targetObject, this); } @Override public void copyObject(Object sourceObject, Object targetObject) { - if (accessKind == OBJECT_ACCESS) { - UNSAFE.putObject(targetObject, fieldOffset, UNSAFE.getObject(sourceObject, fieldOffset)); - } else { - putObject(targetObject, getObject(sourceObject)); - } + copyObjectField(fieldOffset, accessKind, sourceObject, targetObject, this); } } diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/PlatformStringUtils.java b/java/fory-core/src/main/java/org/apache/fory/serializer/PlatformStringUtils.java index 3614e817be..a5f3747b31 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/PlatformStringUtils.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/PlatformStringUtils.java @@ -89,7 +89,7 @@ private static StringFields stringFields() { countOffset = UNSAFE.objectFieldOffset(countField); offsetOffset = UNSAFE.objectFieldOffset(offsetField); } - long coderOffset = valueFieldIsBytes ? StringCoderField.OFFSET : -1; + long coderOffset = valueFieldIsBytes ? stringCoderFieldOffset() : -1; return new StringFields( true, valueFieldIsChars, @@ -112,15 +112,11 @@ private static Field getStringFieldNullable(String fieldName) { } } - private static class StringCoderField { - private static final long OFFSET; - - static { - try { - OFFSET = UNSAFE.objectFieldOffset(String.class.getDeclaredField("coder")); - } catch (NoSuchFieldException e) { - throw new RuntimeException(e); - } + private static long stringCoderFieldOffset() { + try { + return UNSAFE.objectFieldOffset(String.class.getDeclaredField("coder")); + } catch (NoSuchFieldException e) { + throw new RuntimeException(e); } } diff --git a/java/fory-core/src/main/java25/org/apache/fory/platform/internal/_UnsafeUtils.java b/java/fory-core/src/main/java25/org/apache/fory/platform/internal/_UnsafeUtils.java new file mode 100644 index 0000000000..63ecfd36ce --- /dev/null +++ b/java/fory-core/src/main/java25/org/apache/fory/platform/internal/_UnsafeUtils.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.platform.internal; + +/** JDK25 replacement that removes the root runtime's Unsafe descriptor from the class graph. */ +// CHECKSTYLE.OFF:TypeName +public final class _UnsafeUtils { + // CHECKSTYLE.ON:TypeName + private _UnsafeUtils() {} +} diff --git a/java/fory-core/src/test/java/org/apache/fory/builder/BuilderUnsafeClassGraphTest.java b/java/fory-core/src/test/java/org/apache/fory/builder/Jdk25UnsafeClassGraphTest.java similarity index 80% rename from java/fory-core/src/test/java/org/apache/fory/builder/BuilderUnsafeClassGraphTest.java rename to java/fory-core/src/test/java/org/apache/fory/builder/Jdk25UnsafeClassGraphTest.java index 9a76cf607a..9a3879057d 100644 --- a/java/fory-core/src/test/java/org/apache/fory/builder/BuilderUnsafeClassGraphTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/builder/Jdk25UnsafeClassGraphTest.java @@ -32,15 +32,17 @@ import java.util.stream.Stream; import org.testng.annotations.Test; -public class BuilderUnsafeClassGraphTest { - private static final Path ROOT_BUILDER = Paths.get("src/main/java/org/apache/fory/builder"); - private static final Path JAVA25_BUILDER = Paths.get("src/main/java25/org/apache/fory/builder"); +public class Jdk25UnsafeClassGraphTest { + private static final Path ROOT_SOURCES = Paths.get("src/main/java/org/apache/fory"); + private static final Path JAVA25_SOURCES = Paths.get("src/main/java25/org/apache/fory"); private static final Pattern ROOT_UNSAFE_REFERENCE = Pattern.compile( "import\\s+sun\\.misc\\.Unsafe|" + "sun\\.misc\\.Unsafe|" + "\\bUnsafe\\.class\\b|" + + "_UnsafeUtils\\.UNSAFE|" + "_JDKAccess\\.UNSAFE|" + + "Class\\.forName\\(\"sun\\.misc\\.Unsafe\"\\)|" + "TypeRef\\.of\\(Unsafe"); private static final Pattern JAVA25_UNSAFE_REFERENCE = Pattern.compile( @@ -54,7 +56,7 @@ public class BuilderUnsafeClassGraphTest { @Test public void testUnsafeOwner() throws IOException { List violations = new ArrayList<>(); - try (Stream paths = Files.walk(ROOT_BUILDER)) { + try (Stream paths = Files.walk(ROOT_SOURCES)) { paths .filter(path -> path.toString().endsWith(".java")) .forEach( @@ -64,8 +66,8 @@ public void testUnsafeOwner() throws IOException { if (!ROOT_UNSAFE_REFERENCE.matcher(source).find()) { return; } - Path relative = ROOT_BUILDER.relativize(path); - Path replacement = JAVA25_BUILDER.resolve(relative); + Path relative = ROOT_SOURCES.relativize(path); + Path replacement = JAVA25_SOURCES.resolve(relative); if (!Files.exists(replacement)) { violations.add(relative.toString().replace('\\', '/')); } @@ -76,13 +78,13 @@ public void testUnsafeOwner() throws IOException { } assertTrue( violations.isEmpty(), - "Root builder classes that mention Unsafe must have Java 25 replacements: " + violations); + "Root classes that mention Unsafe must have Java 25 replacements: " + violations); } @Test public void testJava25OwnerIsClean() throws IOException { List violations = new ArrayList<>(); - try (Stream paths = Files.walk(JAVA25_BUILDER)) { + try (Stream paths = Files.walk(JAVA25_SOURCES)) { paths .filter(path -> path.toString().endsWith(".java")) .forEach( @@ -90,7 +92,7 @@ public void testJava25OwnerIsClean() throws IOException { try { String source = new String(Files.readAllBytes(path), StandardCharsets.UTF_8); if (JAVA25_UNSAFE_REFERENCE.matcher(source).find()) { - violations.add(JAVA25_BUILDER.relativize(path).toString().replace('\\', '/')); + violations.add(JAVA25_SOURCES.relativize(path).toString().replace('\\', '/')); } } catch (IOException e) { throw new RuntimeException(e); @@ -99,6 +101,6 @@ public void testJava25OwnerIsClean() throws IOException { } assertTrue( violations.isEmpty(), - "Java 25 builder replacements must not reference sun.misc.Unsafe: " + violations); + "Java 25 replacements must not reference sun.misc.Unsafe: " + violations); } } From a9c29fdaa09833d0b34f63d9490a74e2d40989fe Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Tue, 2 Jun 2026 15:56:35 +0800 Subject: [PATCH 47/69] fix(java): repair allocator diagnostics and native image init --- .../apache/fory/reflect/ObjectCreators.java | 18 ++++++++++++++++-- .../fory-core/native-image.properties | 4 ++++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/java/fory-core/src/main/java/org/apache/fory/reflect/ObjectCreators.java b/java/fory-core/src/main/java/org/apache/fory/reflect/ObjectCreators.java index 9a4842a27f..108f2c01cb 100644 --- a/java/fory-core/src/main/java/org/apache/fory/reflect/ObjectCreators.java +++ b/java/fory-core/src/main/java/org/apache/fory/reflect/ObjectCreators.java @@ -25,6 +25,7 @@ import java.lang.invoke.MethodType; import java.lang.reflect.Constructor; import java.lang.reflect.Field; +import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.lang.reflect.Modifier; import java.util.ArrayList; @@ -343,8 +344,21 @@ private static MethodHandle constructorHandle(Class type, Constructor cons } } - private static ForyException makeException(Class type, Throwable cause) { - return new ForyException("Failed to create instance for " + type, cause); + private static RuntimeException makeException(Class type, Throwable cause) { + Throwable target = unwrapConstructorFailure(cause); + // Keep constructor invocation failures outside ForyException so top-level deserialization can + // attach the read-object trail before surfacing the error to users. + return new RuntimeException("Failed to create instance for " + type, target); + } + + private static Throwable unwrapConstructorFailure(Throwable cause) { + if (cause instanceof InvocationTargetException) { + Throwable target = ((InvocationTargetException) cause).getTargetException(); + if (target != null) { + return target; + } + } + return cause; } private static final class ReflectiveNoArgCtrObjectCreator extends ObjectCreator { diff --git a/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties b/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties index 4a135008bb..d0586147a2 100644 --- a/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties +++ b/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties @@ -30,11 +30,15 @@ Args=--initialize-at-build-time=org.apache.fory.annotation.ForyField$Dynamic,\ org.apache.fory.memory.MemoryBuffer,\ org.apache.fory.memory.NativeByteOrder,\ org.apache.fory.platform.AndroidSupport,\ + org.apache.fory.platform.internal._UnsafeUtils,\ + org.apache.fory.reflect.ConstructorBypassAllocator,\ org.apache.fory.reflect.ObjectCreatorRegistry,\ org.apache.fory.reflect.JvmTypeUseMetadata,\ org.apache.fory.reflect.TypeUseMetadata,\ org.apache.fory.resolver.StaticGeneratedSerializerRegistry,\ + org.apache.fory.serializer.collection.CollectionSerializers,\ org.apache.fory.serializer.collection.GuavaCollectionSerializers,\ + org.apache.fory.serializer.collection.MapSerializers,\ org.apache.fory.serializer.struct.Fingerprint,\ org.apache.fory.serializer.Float16Serializer,\ org.apache.fory.serializer.PlatformStringUtils,\ From 73fce7d1aebc96597d93ab28f85ce5407e6cf1ce Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Tue, 2 Jun 2026 16:09:29 +0800 Subject: [PATCH 48/69] test(java): verify jdk25 multi-release class graph --- java/fory-core/pom.xml | 136 +------------- .../fory/reflect/FieldAccessorStrategy.java | 22 +-- .../fory/reflect/FieldAccessorStrategy.java | 24 +-- .../builder/Jdk25MultiReleaseJarVerifier.java | 177 ++++++++++++++++++ 4 files changed, 208 insertions(+), 151 deletions(-) create mode 100644 java/fory-core/src/test/java/org/apache/fory/builder/Jdk25MultiReleaseJarVerifier.java diff --git a/java/fory-core/pom.xml b/java/fory-core/pom.xml index b43f911ef0..7eb856d903 100644 --- a/java/fory-core/pom.xml +++ b/java/fory-core/pom.xml @@ -423,135 +423,15 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + - - - - - - - - - + diff --git a/java/fory-core/src/main/java/org/apache/fory/reflect/FieldAccessorStrategy.java b/java/fory-core/src/main/java/org/apache/fory/reflect/FieldAccessorStrategy.java index 4028919cfd..038112830b 100644 --- a/java/fory-core/src/main/java/org/apache/fory/reflect/FieldAccessorStrategy.java +++ b/java/fory-core/src/main/java/org/apache/fory/reflect/FieldAccessorStrategy.java @@ -135,11 +135,11 @@ private static void copyObjectField( } } - private abstract static class UnsafeAccessor extends FieldAccessor { + private abstract static class InstanceAccessor extends FieldAccessor { protected final long fieldOffset; private final int accessKind; - UnsafeAccessor(Field field) { + InstanceAccessor(Field field) { super(field); fieldOffset = fieldOffset(field); accessKind = accessKind(field); @@ -188,7 +188,7 @@ static FieldAccessor createStaticAccessor(Field field) { } /** Primitive boolean accessor. */ - public static class BooleanAccessor extends UnsafeAccessor { + public static class BooleanAccessor extends InstanceAccessor { public BooleanAccessor(Field field) { super(field); Preconditions.checkArgument(field.getType() == boolean.class); @@ -218,7 +218,7 @@ public void putBoolean(Object obj, boolean value) { } /** Primitive byte accessor. */ - public static class ByteAccessor extends UnsafeAccessor { + public static class ByteAccessor extends InstanceAccessor { public ByteAccessor(Field field) { super(field); Preconditions.checkArgument(field.getType() == byte.class); @@ -248,7 +248,7 @@ public void putByte(Object obj, byte value) { } /** Primitive char accessor. */ - public static class CharAccessor extends UnsafeAccessor { + public static class CharAccessor extends InstanceAccessor { public CharAccessor(Field field) { super(field); Preconditions.checkArgument(field.getType() == char.class); @@ -278,7 +278,7 @@ public void putChar(Object obj, char value) { } /** Primitive short accessor. */ - public static class ShortAccessor extends UnsafeAccessor { + public static class ShortAccessor extends InstanceAccessor { public ShortAccessor(Field field) { super(field); Preconditions.checkArgument(field.getType() == short.class); @@ -308,7 +308,7 @@ public void putShort(Object obj, short value) { } /** Primitive int accessor. */ - public static class IntAccessor extends UnsafeAccessor { + public static class IntAccessor extends InstanceAccessor { public IntAccessor(Field field) { super(field); Preconditions.checkArgument(field.getType() == int.class); @@ -338,7 +338,7 @@ public void putInt(Object obj, int value) { } /** Primitive long accessor. */ - public static class LongAccessor extends UnsafeAccessor { + public static class LongAccessor extends InstanceAccessor { public LongAccessor(Field field) { super(field); Preconditions.checkArgument(field.getType() == long.class); @@ -368,7 +368,7 @@ public void putLong(Object obj, long value) { } /** Primitive float accessor. */ - public static class FloatAccessor extends UnsafeAccessor { + public static class FloatAccessor extends InstanceAccessor { public FloatAccessor(Field field) { super(field); Preconditions.checkArgument(field.getType() == float.class); @@ -398,7 +398,7 @@ public void putFloat(Object obj, float value) { } /** Primitive double accessor. */ - public static class DoubleAccessor extends UnsafeAccessor { + public static class DoubleAccessor extends InstanceAccessor { public DoubleAccessor(Field field) { super(field); Preconditions.checkArgument(field.getType() == double.class); @@ -428,7 +428,7 @@ public void putDouble(Object obj, double value) { } /** Object accessor. */ - public static class ObjectAccessor extends UnsafeAccessor { + public static class ObjectAccessor extends InstanceAccessor { public ObjectAccessor(Field field) { super(field); Preconditions.checkArgument(!TypeUtils.isPrimitive(field.getType())); diff --git a/java/fory-core/src/main/java25/org/apache/fory/reflect/FieldAccessorStrategy.java b/java/fory-core/src/main/java25/org/apache/fory/reflect/FieldAccessorStrategy.java index afdedb918b..06bae681f1 100644 --- a/java/fory-core/src/main/java25/org/apache/fory/reflect/FieldAccessorStrategy.java +++ b/java/fory-core/src/main/java25/org/apache/fory/reflect/FieldAccessorStrategy.java @@ -142,13 +142,13 @@ private static RuntimeException accessorFailure(Field field, Throwable cause) { return new RuntimeException("Failed to access field: " + field, cause); } - private abstract static class VarHandleAccessor extends FieldAccessor { + private abstract static class InstanceAccessor extends FieldAccessor { protected final VarHandle handle; protected final boolean isStatic; protected final boolean isFinal; protected volatile Field finalField; - VarHandleAccessor(Field field) { + InstanceAccessor(Field field) { super(field); handle = fieldHandle(field); isStatic = Modifier.isStatic(field.getModifiers()); @@ -268,7 +268,7 @@ protected void setFinalDouble(Object obj, double value, Throwable cause) { } /** Primitive boolean accessor. */ - public static class BooleanAccessor extends VarHandleAccessor { + public static class BooleanAccessor extends InstanceAccessor { public BooleanAccessor(Field field) { super(field); Preconditions.checkArgument(field.getType() == boolean.class); @@ -313,7 +313,7 @@ public void putBoolean(Object obj, boolean value) { } /** Primitive byte accessor. */ - public static class ByteAccessor extends VarHandleAccessor { + public static class ByteAccessor extends InstanceAccessor { public ByteAccessor(Field field) { super(field); Preconditions.checkArgument(field.getType() == byte.class); @@ -358,7 +358,7 @@ public void putByte(Object obj, byte value) { } /** Primitive char accessor. */ - public static class CharAccessor extends VarHandleAccessor { + public static class CharAccessor extends InstanceAccessor { public CharAccessor(Field field) { super(field); Preconditions.checkArgument(field.getType() == char.class); @@ -403,7 +403,7 @@ public void putChar(Object obj, char value) { } /** Primitive short accessor. */ - public static class ShortAccessor extends VarHandleAccessor { + public static class ShortAccessor extends InstanceAccessor { public ShortAccessor(Field field) { super(field); Preconditions.checkArgument(field.getType() == short.class); @@ -448,7 +448,7 @@ public void putShort(Object obj, short value) { } /** Primitive int accessor. */ - public static class IntAccessor extends VarHandleAccessor { + public static class IntAccessor extends InstanceAccessor { public IntAccessor(Field field) { super(field); Preconditions.checkArgument(field.getType() == int.class); @@ -493,7 +493,7 @@ public void putInt(Object obj, int value) { } /** Primitive long accessor. */ - public static class LongAccessor extends VarHandleAccessor { + public static class LongAccessor extends InstanceAccessor { public LongAccessor(Field field) { super(field); Preconditions.checkArgument(field.getType() == long.class); @@ -538,7 +538,7 @@ public void putLong(Object obj, long value) { } /** Primitive float accessor. */ - public static class FloatAccessor extends VarHandleAccessor { + public static class FloatAccessor extends InstanceAccessor { public FloatAccessor(Field field) { super(field); Preconditions.checkArgument(field.getType() == float.class); @@ -583,7 +583,7 @@ public void putFloat(Object obj, float value) { } /** Primitive double accessor. */ - public static class DoubleAccessor extends VarHandleAccessor { + public static class DoubleAccessor extends InstanceAccessor { public DoubleAccessor(Field field) { super(field); Preconditions.checkArgument(field.getType() == double.class); @@ -628,7 +628,7 @@ public void putDouble(Object obj, double value) { } /** Object accessor. */ - public static class ObjectAccessor extends VarHandleAccessor { + public static class ObjectAccessor extends InstanceAccessor { public ObjectAccessor(Field field) { super(field); Preconditions.checkArgument(!TypeUtils.isPrimitive(field.getType())); @@ -669,7 +669,7 @@ static final class StaticObjectAccessor extends ObjectAccessor { } } - static final class GeneratedAccessor extends VarHandleAccessor { + static final class GeneratedAccessor extends InstanceAccessor { GeneratedAccessor(Field field) { super(field); } diff --git a/java/fory-core/src/test/java/org/apache/fory/builder/Jdk25MultiReleaseJarVerifier.java b/java/fory-core/src/test/java/org/apache/fory/builder/Jdk25MultiReleaseJarVerifier.java new file mode 100644 index 0000000000..fc59c7c806 --- /dev/null +++ b/java/fory-core/src/test/java/org/apache/fory/builder/Jdk25MultiReleaseJarVerifier.java @@ -0,0 +1,177 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.builder; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Enumeration; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.jar.JarEntry; +import java.util.jar.JarFile; + +/** Verifies the packaged JDK25 multi-release class graph. */ +public final class Jdk25MultiReleaseJarVerifier { + private static final String VERSION_25_PREFIX = "META-INF/versions/25/"; + private static final String FORY_CLASS_PREFIX = "org/apache/fory/"; + private static final String SHADED_CLASS_PREFIX = "org/apache/fory/shaded/"; + private static final String[] FORBIDDEN_CONSTANTS = { + "sun/misc/Unsafe", "sun.misc.Unsafe", "jdk/internal/reflect", "jdk.internal.reflect" + }; + private static final String[] REQUIRED_VERSION_25_CLASSES = { + "module-info.class", + "org/apache/fory/memory/LittleEndian.class", + "org/apache/fory/memory/MemoryBuffer.class", + "org/apache/fory/platform/internal/DefineClass.class", + "org/apache/fory/platform/internal/_Lookup.class", + "org/apache/fory/platform/internal/_UnsafeUtils.class", + "org/apache/fory/builder/UnsafeCodegenSupport.class", + "org/apache/fory/reflect/FieldAccessorStrategy.class", + "org/apache/fory/reflect/HiddenFieldAccessorFactory.class", + "org/apache/fory/reflect/ConstructorBypassAllocator.class", + "org/apache/fory/serializer/PlatformStringUtils.class" + }; + + private Jdk25MultiReleaseJarVerifier() {} + + public static void main(String[] args) throws IOException { + if (args.length != 1) { + throw new IllegalArgumentException("Usage: Jdk25MultiReleaseJarVerifier "); + } + verify(Paths.get(args[0])); + } + + static void verify(Path jarPath) throws IOException { + Map rootClasses = new HashMap<>(); + Map version25Classes = new HashMap<>(); + List violations = new ArrayList<>(); + try (JarFile jarFile = new JarFile(jarPath.toFile())) { + Enumeration entries = jarFile.entries(); + while (entries.hasMoreElements()) { + JarEntry entry = entries.nextElement(); + if (entry.isDirectory() || !entry.getName().endsWith(".class")) { + continue; + } + String name = entry.getName(); + if (isForyRootClass(name)) { + rootClasses.put(name, readFully(jarFile, entry)); + } else if (name.startsWith(VERSION_25_PREFIX)) { + String className = name.substring(VERSION_25_PREFIX.length()); + if ("module-info.class".equals(className) || isForyRootClass(className)) { + version25Classes.put(className, readFully(jarFile, entry)); + } + } + } + } + + verifyRequiredClasses(rootClasses, version25Classes, violations); + verifyForbiddenConstants(rootClasses, version25Classes, violations); + if (!violations.isEmpty()) { + throw new AssertionError( + "Invalid JDK25 multi-release fory-core jar " + jarPath + ": " + violations); + } + } + + private static boolean isForyRootClass(String name) { + return name.startsWith(FORY_CLASS_PREFIX) && !name.startsWith(SHADED_CLASS_PREFIX); + } + + private static void verifyRequiredClasses( + Map rootClasses, Map version25Classes, List out) { + if (rootClasses.containsKey("org/apache/fory/platform/UnsafeOps.class")) { + out.add("Root UnsafeOps class must not be packaged"); + } + if (version25Classes.containsKey("org/apache/fory/platform/UnsafeOps.class")) { + out.add("JDK25 UnsafeOps class must not be packaged"); + } + for (String requiredClass : REQUIRED_VERSION_25_CLASSES) { + if (!version25Classes.containsKey(requiredClass)) { + out.add("Missing JDK25 multi-release class " + requiredClass); + } + } + } + + private static void verifyForbiddenConstants( + Map rootClasses, Map version25Classes, List out) { + for (Map.Entry entry : rootClasses.entrySet()) { + if (containsForbiddenConstant(entry.getValue()) + && !version25Classes.containsKey(entry.getKey())) { + out.add( + "Root class " + entry.getKey() + " has forbidden constants without JDK25 replacement"); + } + } + Set resolvedClasses = new HashSet<>(rootClasses.keySet()); + resolvedClasses.addAll(version25Classes.keySet()); + resolvedClasses.remove("module-info.class"); + for (String className : resolvedClasses) { + byte[] bytes = + version25Classes.containsKey(className) + ? version25Classes.get(className) + : rootClasses.get(className); + if (containsForbiddenConstant(bytes)) { + out.add("JDK25 resolved class " + className + " has forbidden constants"); + } + } + } + + private static boolean containsForbiddenConstant(byte[] bytes) { + for (String forbiddenConstant : FORBIDDEN_CONSTANTS) { + if (containsAscii(bytes, forbiddenConstant)) { + return true; + } + } + return false; + } + + private static boolean containsAscii(byte[] bytes, String value) { + byte[] target = value.getBytes(StandardCharsets.US_ASCII); + int maxStart = bytes.length - target.length; + for (int i = 0; i <= maxStart; i++) { + int j = 0; + while (j < target.length && bytes[i + j] == target[j]) { + j++; + } + if (j == target.length) { + return true; + } + } + return false; + } + + private static byte[] readFully(JarFile jarFile, JarEntry entry) throws IOException { + try (InputStream inputStream = jarFile.getInputStream(entry)) { + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + byte[] buffer = new byte[8192]; + int read; + while ((read = inputStream.read(buffer)) >= 0) { + outputStream.write(buffer, 0, read); + } + return outputStream.toByteArray(); + } + } +} From 6095e05738d371228d974d04068afe864868b92b Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Tue, 2 Jun 2026 16:50:22 +0800 Subject: [PATCH 49/69] fix(java): preserve trusted lookup field writes --- .agents/languages/java.md | 19 +- ci/run_ci.sh | 7 +- docs/guide/java/native-serialization.md | 4 +- docs/guide/java/troubleshooting.md | 16 +- docs/guide/kotlin/configuration.md | 2 +- integration_tests/jpms_tests/pom.xml | 11 +- .../JpmsFieldAccessorTest.java | 47 ++ .../apache/fory/reflect/FieldAccessor.java | 4 - .../fory/reflect/FieldAccessorFactory.java | 33 -- .../fory/reflect/FieldAccessorStrategy.java | 27 - .../scala/SingletonCollectionSerializer.java | 27 +- .../scala/SingletonMapSerializer.java | 27 +- .../scala/SingletonObjectSerializer.java | 27 +- .../fory/platform/internal/_Lookup.java | 35 +- .../fory/reflect/FieldAccessorStrategy.java | 506 ++---------------- .../test/java/org/apache/fory/TestUtils.java | 6 +- 16 files changed, 211 insertions(+), 587 deletions(-) diff --git a/.agents/languages/java.md b/.agents/languages/java.md index ad37d7562a..8c31c89fbb 100644 --- a/.agents/languages/java.md +++ b/.agents/languages/java.md @@ -60,8 +60,8 @@ Load this file when changing anything under `java/` or when Java drives a cross- - Put latest-JDK or virtual-thread tests in the latest-JDK test modules with the matching compiler/profile floor, and centralize runtime-version probing in existing compatibility utilities. - For JDK25+ zero-Unsafe work, preserve serializer-family selection by type and configuration. Do not switch a type from `ObjectStreamSerializer` or another Fory serializer family to `JavaSerializer`, a JDK stream fallback, or any broad `java.* Serializable` fallback by JDK version or no-arg-constructor shape. - JDK25+ zero-Unsafe runtime support is a JPMS named-module design. Fory core must run with `--add-opens=java.base/java.lang.invoke=org.apache.fory.core` so it can obtain the trusted lookup; missing this open is an invalid access configuration, not a reason to open per-package JDK internals or switch serializer/object-creation families. Do not use `ALL-UNNAMED` as the zero-Unsafe verification target. -- Keep JDK25+ unsafe-removal implementation invariants in agent/design docs and tests, not user guides. User guides should document user actions such as `--sun-misc-unsafe-memory-access=deny`, `java.base/java.lang.invoke` opens, JDK26+ `--enable-final-field-mutation`, package opens for user named modules, and public APIs such as `@ForyConstructor` and `BaseFory.registerConstructor(...)`; do not expose internal serializer names, owner-model rationale, or avoided fallback strategies there. -- For JDK25+ zero-Unsafe final-field behavior, distinguish JDK25 from JDK26+: JDK25 has no final-field mutation flag requirement, while JDK26+ requires `--enable-final-field-mutation` for post-construction final-field writes. +- Keep JDK25+ unsafe-removal implementation invariants in agent/design docs and tests, not user guides. User guides should document user actions such as `--sun-misc-unsafe-memory-access=deny`, `java.base/java.lang.invoke` opens, package opens for user named modules, and public APIs such as `@ForyConstructor` and `BaseFory.registerConstructor(...)`; do not expose internal serializer names, owner-model rationale, or avoided fallback strategies there. +- JDK25+ zero-Unsafe final-field writes must use a true target-class trusted lookup from the original `IMPL_LOOKUP`, not `IMPL_LOOKUP.in(type)`. JDK26+ normal Fory final-field restoration must pass with `--illegal-final-field-mutation=deny` and must not require `--enable-final-field-mutation`. - For JDK25+ object creation, do not use `sun.reflect.ReflectionFactory` or `jdk.internal.reflect.ReflectionFactory`. The shared `ObjectCreators` facade should route ObjectStream-compatible construction through `ParentNoArgCtrObjectCreator`; the replaceable constructor-bypass allocator owns the JDK8-24 Unsafe allocation path and the JDK25+ `ObjectStreamClass.newInstance` path. Serializable classes without a no-arg constructor may use that trusted-lookup owner; non-Serializable classes without a no-arg constructor require `@ForyConstructor`, `BaseFory.registerConstructor(...)`, record construction, or a custom serializer. - In JDK25+ constructor-bypass allocation, the allocator is per type: pass the target class to the allocator constructor, cache `ObjectStreamClass.lookupAny(...)` in that instance, and expose an instance `allocate()` method. Let `ObjectStreamClass.newInstance` validate unsupported classes. Do not add redundant `Serializable` prechecks or per-call descriptor lookups, and keep exception/message construction in cold helper methods. - Keep the Java25 `_Lookup` and `DefineClass` overlays unless a future refactor can merge them without exposing Unsafe to the JDK25 class graph or replacing direct hidden-class APIs with reflective wrappers. Root `_Lookup` uses Unsafe for the JDK8-24 trusted-lookup fast path, while Java25 `_Lookup` uses the required `java.lang.invoke` open. Root `DefineClass` targets Java 8 bytecode and cannot directly reference `Lookup#defineHiddenClass` or `Lookup.ClassOption.NESTMATE`; Java25 `DefineClass` owns that direct API use. @@ -86,8 +86,9 @@ Load this file when changing anything under `java/` or when Java drives a cross- - Multi-release replacement is class-file exact. Replacing `Foo.class` does not replace `Foo$Bar.class`; root nested classes that contain Unsafe descriptors must either be eliminated, moved under an exact replaceable owner, or have their direct Unsafe operations moved to the - already-overlaid top-level class. JDK25 package checks must run class-level `jdeps` over Fory - classes so nested leaks are caught without rejecting shaded third-party benchmark dependencies. + already-overlaid top-level class. JDK25 package checks must enumerate packaged Fory class files + and inspect forbidden constants so nested leaks are caught without rejecting shaded third-party + dependencies. - Java 25 multi-release classes never run on Android. Do not keep `AndroidSupport.IS_ANDROID` branches or imports under `src/main/java25`; Android compatibility belongs to the root sources. - String zero-copy construction is serializer-owned behavior. Keep private `String` constructor @@ -101,6 +102,16 @@ Load this file when changing anything under `java/` or when Java drives a cross- - JDK25+ `PlatformStringUtils` getter methods sit behind `StringSerializer` static-final access gates. Do not add per-call access checks in those getters; missing module opens should fail at trusted-lookup initialization or cold setup, not inside string hot paths. +- JDK25+ `FieldAccessorStrategy` owns instance field access only. Do not add static-field handling + or per-write reflection fallback there, and do not expose public `FieldAccessor` static-field + factories. Static special cases such as Scala `MODULE$` belong to the owning serializer and should + use a cached `_JDKAccess._trustedLookup(...).findStaticGetter(...)` handle at that call site. +- JDK25+ final-instance field mutation should use the same trusted-lookup `VarHandle` owner path as + non-final field mutation. Do not add a final-field-only `MethodHandle` setter path or ordinary + reflective `Field.set*` fallback. +- Do not call `FieldAccessor.checkObj` in the JDK25+ VarHandle field-access hot path. VarHandle + validates null and receiver type itself, while root Unsafe offset access still needs explicit + receiver validation because Unsafe does not. - JDK25+ collection serializers must fail unsupported `Collections.newSetFromMap` backing maps before writing or copying. Do not rewrite them to `HashMap`, because that changes equality semantics and can drop entries. diff --git a/ci/run_ci.sh b/ci/run_ci.sh index 80db64756f..2abdf7157d 100755 --- a/ci/run_ci.sh +++ b/ci/run_ci.sh @@ -107,9 +107,8 @@ jdk25_access_options() { printf " %s" "--add-opens=java.base/java.lang.invoke=${fory_open_targets}" } -jdk26_final_field_options() { - local fory_modules="${1:-org.apache.fory.core}" - printf "%s" "--enable-final-field-mutation=${fory_modules}" +jdk26_final_field_policy_options() { + printf "%s" "--illegal-final-field-mutation=deny" } jdk25_plus_options() { @@ -117,7 +116,7 @@ jdk25_plus_options() { local fory_targets="${2:-org.apache.fory.core}" printf "%s" "$(jdk25_access_options "$fory_targets")" if [[ "$java_major" -ge 26 ]]; then - printf " %s" "$(jdk26_final_field_options "$fory_targets")" + printf " %s" "$(jdk26_final_field_policy_options)" fi } diff --git a/docs/guide/java/native-serialization.md b/docs/guide/java/native-serialization.md index b3596ce7ee..23a24fca5d 100644 --- a/docs/guide/java/native-serialization.md +++ b/docs/guide/java/native-serialization.md @@ -205,8 +205,8 @@ object creation and field setting. On JDK25+ with Unsafe memory access denied, F if the class cannot be created by supported Java mechanisms. Use `@ForyConstructor`, `registerConstructor(...)`, a record canonical constructor, or a custom serializer for those classes. Use the `java.base/java.lang.invoke` open shown in troubleshooting for supported JDK25+ access paths. -On JDK26+, enable final-field mutation for the Fory runtime module. JDK25 does not have the final-field -mutation flag. See +Fory does not require `--enable-final-field-mutation` for ordinary final-field restoration on +JDK26+. See [Troubleshooting](troubleshooting.md#jdk25-zero-unsafe-mode-and-module-opens) for the required JVM flags. diff --git a/docs/guide/java/troubleshooting.md b/docs/guide/java/troubleshooting.md index dbbb2e509b..394677d605 100644 --- a/docs/guide/java/troubleshooting.md +++ b/docs/guide/java/troubleshooting.md @@ -166,18 +166,10 @@ module: If this open is missing, Fory reports an error that names `java.base/java.lang.invoke`. -On JDK26 and later, normal classes with final instance fields require final-field mutation to be -enabled for the module that contains Fory's mutating code when Unsafe allocation is denied. Use the -Fory module name on the module path: - -```bash ---enable-final-field-mutation=org.apache.fory.core -``` - -Fory can restore those final fields when final-field mutation is enabled. JDK25 has no -`--enable-final-field-mutation` option, so no final-field mutation flag is needed on JDK25. Named -application modules that contain private fields still need to open the application package to -`org.apache.fory.core`. +Fory does not require `--enable-final-field-mutation` for normal final-field restoration on JDK26 +and later. With the `java.base/java.lang.invoke` open above, Fory uses trusted lookup field handles +instead of ordinary reflective final-field mutation. Named application modules that contain private +fields still need to open the application package to `org.apache.fory.core`. The vectorized Arrow APIs in `fory-format` depend on Apache Arrow's memory layer. With the current Arrow dependency, those APIs are unavailable when `--sun-misc-unsafe-memory-access=deny` is set diff --git a/docs/guide/kotlin/configuration.md b/docs/guide/kotlin/configuration.md index eb37157c2c..b4b2880551 100644 --- a/docs/guide/kotlin/configuration.md +++ b/docs/guide/kotlin/configuration.md @@ -119,7 +119,7 @@ KSP-generated `@ForyStruct` serializers that call a primary constructor require `@ForyConstructor` mapping. Mutable no-argument `@ForyStruct` classes can instead expose serialized `var` properties with `@ForyField`. -The JVM also needs the module opens and final-field mutation option listed in +The JVM also needs the module opens listed in [Java Troubleshooting](../java/troubleshooting.md#jdk25-zero-unsafe-mode-and-module-opens). Common options for Kotlin native-mode payloads: diff --git a/integration_tests/jpms_tests/pom.xml b/integration_tests/jpms_tests/pom.xml index 15a92a1e86..dbb03218e3 100644 --- a/integration_tests/jpms_tests/pom.xml +++ b/integration_tests/jpms_tests/pom.xml @@ -36,7 +36,7 @@ 11 11 UTF-8 - + @@ -87,7 +87,8 @@ --sun-misc-unsafe-memory-access=deny --add-opens=java.base/java.lang.invoke=org.apache.fory.core - ${fory.final.field.mutation.arg} + --add-opens=org.apache.fory.core/org.apache.fory.platform.internal=org.apache.fory.integration_tests + ${fory.final.field.policy.arg} @@ -100,9 +101,9 @@ [26,) - - --enable-final-field-mutation=org.apache.fory.core - + + --illegal-final-field-mutation=deny + diff --git a/integration_tests/jpms_tests/src/test/java/org/apache/fory/integration_tests/JpmsFieldAccessorTest.java b/integration_tests/jpms_tests/src/test/java/org/apache/fory/integration_tests/JpmsFieldAccessorTest.java index c1908ec481..b2cc8299b8 100644 --- a/integration_tests/jpms_tests/src/test/java/org/apache/fory/integration_tests/JpmsFieldAccessorTest.java +++ b/integration_tests/jpms_tests/src/test/java/org/apache/fory/integration_tests/JpmsFieldAccessorTest.java @@ -19,7 +19,12 @@ package org.apache.fory.integration_tests; +import java.lang.invoke.MethodHandle; +import java.lang.invoke.MethodHandles; +import java.lang.invoke.MethodType; +import java.lang.invoke.VarHandle; import java.lang.reflect.Field; +import java.lang.reflect.Method; import org.apache.fory.Fory; import org.apache.fory.integration_tests.constructor.PrivateConstructorBean; import org.apache.fory.integration_tests.model.PrivateFieldBean; @@ -30,6 +35,8 @@ import org.testng.annotations.Test; public class JpmsFieldAccessorTest { + private static final int JDK_MAJOR_VERSION = Runtime.version().feature(); + @Test public void testPrivateFieldAccess() throws Exception { PrivateFieldBean bean = new PrivateFieldBean(7); @@ -49,6 +56,36 @@ public void testPrivateFinalFieldSerialization() { Assert.assertEquals(result.value(), 13); } + @Test + public void testTrustedLookupFinalWrite() throws Throwable { + PrivateFieldBean bean = new PrivateFieldBean(17); + MethodHandles.Lookup lookup = trustedLookup(PrivateFieldBean.class); + VarHandle handle = lookup.findVarHandle(PrivateFieldBean.class, "value", int.class); + handle.set(bean, 19); + Assert.assertEquals(bean.value(), 19); + + MethodHandle setter = + lookup.findSetter(PrivateFieldBean.class, "value", int.class).asType(setterType()); + setter.invokeExact(bean, 23); + Assert.assertEquals(bean.value(), 23); + } + + @Test + public void testReflectionFinalWriteDenied() throws Exception { + if (JDK_MAJOR_VERSION < 26) { + return; + } + PrivateFieldBean bean = new PrivateFieldBean(29); + Field field = PrivateFieldBean.class.getDeclaredField("value"); + field.setAccessible(true); + try { + field.setInt(bean, 31); + Assert.fail("JDK26 denial mode should reject ordinary final-field reflection writes"); + } catch (IllegalAccessException | RuntimeException expected) { + Assert.assertEquals(bean.value(), 29); + } + } + @Test public void testPrivateConstructorBinding() { Fory fory = @@ -68,4 +105,14 @@ public void testPublicSerializerInExportedPackage() { (PublicSerializerValue) fory.deserialize(fory.serialize(new PublicSerializerValue(11))); Assert.assertEquals(result.value, 11); } + + private static MethodHandles.Lookup trustedLookup(Class type) throws Exception { + Class jdkAccess = Class.forName("org.apache.fory.platform.internal._JDKAccess"); + Method method = jdkAccess.getMethod("_trustedLookup", Class.class); + return (MethodHandles.Lookup) method.invoke(null, type); + } + + private static MethodType setterType() { + return MethodType.methodType(void.class, PrivateFieldBean.class, int.class); + } } diff --git a/java/fory-core/src/main/java/org/apache/fory/reflect/FieldAccessor.java b/java/fory-core/src/main/java/org/apache/fory/reflect/FieldAccessor.java index bfde5ca4fe..88c12d4201 100644 --- a/java/fory-core/src/main/java/org/apache/fory/reflect/FieldAccessor.java +++ b/java/fory-core/src/main/java/org/apache/fory/reflect/FieldAccessor.java @@ -210,8 +210,4 @@ public Object getGetter() { public static FieldAccessor createAccessor(Field field) { return FieldAccessorFactory.createAccessor(field); } - - public static FieldAccessor createStaticAccessor(Field field) { - return FieldAccessorFactory.createStaticAccessor(field); - } } diff --git a/java/fory-core/src/main/java/org/apache/fory/reflect/FieldAccessorFactory.java b/java/fory-core/src/main/java/org/apache/fory/reflect/FieldAccessorFactory.java index 7f1bc6c684..782063fef3 100644 --- a/java/fory-core/src/main/java/org/apache/fory/reflect/FieldAccessorFactory.java +++ b/java/fory-core/src/main/java/org/apache/fory/reflect/FieldAccessorFactory.java @@ -56,15 +56,6 @@ static FieldAccessor createAccessor(Field field) { return FieldAccessorStrategy.createAccessor(field); } - static FieldAccessor createStaticAccessor(Field field) { - Preconditions.checkArgument(Modifier.isStatic(field.getModifiers()), field); - if (AndroidSupport.IS_ANDROID) { - field.setAccessible(true); - return new ReflectiveStaticFieldAccessor(field); - } - return FieldAccessorStrategy.createStaticAccessor(field); - } - private static FieldAccessor createRecordAccessor(Field field) { if (AndroidSupport.IS_ANDROID || GraalvmSupport.IN_GRAALVM_NATIVE_IMAGE) { return new ReflectiveRecordFieldAccessor(field); @@ -304,28 +295,4 @@ public Object get(Object obj) { return getter.apply(obj); } } - - static final class ReflectiveStaticFieldAccessor extends FieldAccessor { - ReflectiveStaticFieldAccessor(Field field) { - super(field); - } - - @Override - public Object get(Object obj) { - try { - return field.get(null); - } catch (IllegalAccessException | IllegalArgumentException e) { - throw new ForyException("Failed to read static field reflectively: " + field, e); - } - } - - @Override - public void set(Object obj, Object value) { - try { - field.set(null, value); - } catch (IllegalAccessException | IllegalArgumentException e) { - throw new ForyException("Failed to write static field reflectively: " + field, e); - } - } - } } diff --git a/java/fory-core/src/main/java/org/apache/fory/reflect/FieldAccessorStrategy.java b/java/fory-core/src/main/java/org/apache/fory/reflect/FieldAccessorStrategy.java index 038112830b..cca430f393 100644 --- a/java/fory-core/src/main/java/org/apache/fory/reflect/FieldAccessorStrategy.java +++ b/java/fory-core/src/main/java/org/apache/fory/reflect/FieldAccessorStrategy.java @@ -182,11 +182,6 @@ static FieldAccessor createAccessor(Field field) { } } - static FieldAccessor createStaticAccessor(Field field) { - Preconditions.checkArgument(Modifier.isStatic(field.getModifiers()), field); - return new StaticObjectAccessor(field); - } - /** Primitive boolean accessor. */ public static class BooleanAccessor extends InstanceAccessor { public BooleanAccessor(Field field) { @@ -447,28 +442,6 @@ public void set(Object obj, Object value) { } } - static final class StaticObjectAccessor extends FieldAccessor { - private final Object base; - private final long offset; - - StaticObjectAccessor(Field field) { - super(field); - Preconditions.checkArgument(!TypeUtils.isPrimitive(field.getType())); - base = UNSAFE.staticFieldBase(field); - offset = UNSAFE.staticFieldOffset(field); - } - - @Override - public Object get(Object obj) { - return UNSAFE.getObject(base, offset); - } - - @Override - public void set(Object obj, Object value) { - UNSAFE.putObject(base, offset, value); - } - } - static final class GeneratedAccessor extends FieldAccessor { private static final ClassValueCache>> cache = ClassValueCache.newClassKeyCache(8); diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/scala/SingletonCollectionSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/scala/SingletonCollectionSerializer.java index e17e309a02..4ed73cceac 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/scala/SingletonCollectionSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/scala/SingletonCollectionSerializer.java @@ -19,17 +19,16 @@ package org.apache.fory.serializer.scala; +import java.lang.invoke.MethodHandle; import java.lang.reflect.Field; import java.util.Collection; import org.apache.fory.context.ReadContext; import org.apache.fory.context.WriteContext; import org.apache.fory.exception.ForyException; import org.apache.fory.platform.AndroidSupport; -import org.apache.fory.platform.GraalvmSupport; -import org.apache.fory.reflect.FieldAccessor; +import org.apache.fory.platform.internal._JDKAccess; import org.apache.fory.resolver.TypeResolver; import org.apache.fory.serializer.collection.CollectionLikeSerializer; -import org.apache.fory.util.Preconditions; /** * Singleton serializer for scala collection. We need this serializer for fory jit serialization, @@ -39,7 +38,7 @@ @SuppressWarnings("rawtypes") public class SingletonCollectionSerializer extends CollectionLikeSerializer { private final Field field; - private FieldAccessor accessor; + private MethodHandle accessor; public SingletonCollectionSerializer(TypeResolver typeResolver, Class cls) { super(typeResolver, cls, false); @@ -77,12 +76,24 @@ public Object read(ReadContext readContext) { throw new ForyException("Failed to read Scala singleton field: " + type, e); } } - FieldAccessor accessor = this.accessor; + MethodHandle accessor = this.accessor; if (accessor == null) { - Preconditions.checkArgument(!GraalvmSupport.isGraalBuildTime()); - accessor = this.accessor = FieldAccessor.createStaticAccessor(field); + accessor = this.accessor = staticGetter(); + } + try { + return accessor.invoke(); + } catch (Throwable e) { + throw new ForyException("Failed to read Scala singleton field: " + type, e); + } + } + + private MethodHandle staticGetter() { + try { + return _JDKAccess._trustedLookup(field.getDeclaringClass()) + .findStaticGetter(field.getDeclaringClass(), field.getName(), field.getType()); + } catch (NoSuchFieldException | IllegalAccessException | RuntimeException e) { + throw new ForyException("Failed to access Scala singleton field: " + type, e); } - return accessor.getObject(null); } @Override diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/scala/SingletonMapSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/scala/SingletonMapSerializer.java index 9ef08ac09b..58813c094b 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/scala/SingletonMapSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/scala/SingletonMapSerializer.java @@ -19,17 +19,16 @@ package org.apache.fory.serializer.scala; +import java.lang.invoke.MethodHandle; import java.lang.reflect.Field; import java.util.Map; import org.apache.fory.context.ReadContext; import org.apache.fory.context.WriteContext; import org.apache.fory.exception.ForyException; import org.apache.fory.platform.AndroidSupport; -import org.apache.fory.platform.GraalvmSupport; -import org.apache.fory.reflect.FieldAccessor; +import org.apache.fory.platform.internal._JDKAccess; import org.apache.fory.resolver.TypeResolver; import org.apache.fory.serializer.collection.MapLikeSerializer; -import org.apache.fory.util.Preconditions; /** * Singleton serializer for scala map. We need this serializer for fory jit serialization, otherwise @@ -39,7 +38,7 @@ @SuppressWarnings("rawtypes") public class SingletonMapSerializer extends MapLikeSerializer { private final Field field; - private FieldAccessor accessor; + private MethodHandle accessor; public SingletonMapSerializer(TypeResolver typeResolver, Class cls) { super(typeResolver, cls, false); @@ -77,12 +76,24 @@ public Object read(ReadContext readContext) { throw new ForyException("Failed to read Scala singleton field: " + type, e); } } - FieldAccessor accessor = this.accessor; + MethodHandle accessor = this.accessor; if (accessor == null) { - Preconditions.checkArgument(!GraalvmSupport.isGraalBuildTime()); - accessor = this.accessor = FieldAccessor.createStaticAccessor(field); + accessor = this.accessor = staticGetter(); + } + try { + return accessor.invoke(); + } catch (Throwable e) { + throw new ForyException("Failed to read Scala singleton field: " + type, e); + } + } + + private MethodHandle staticGetter() { + try { + return _JDKAccess._trustedLookup(field.getDeclaringClass()) + .findStaticGetter(field.getDeclaringClass(), field.getName(), field.getType()); + } catch (NoSuchFieldException | IllegalAccessException | RuntimeException e) { + throw new ForyException("Failed to access Scala singleton field: " + type, e); } - return accessor.getObject(null); } @Override diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/scala/SingletonObjectSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/scala/SingletonObjectSerializer.java index 408da2ebc3..7c62236a5e 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/scala/SingletonObjectSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/scala/SingletonObjectSerializer.java @@ -19,16 +19,15 @@ package org.apache.fory.serializer.scala; +import java.lang.invoke.MethodHandle; import java.lang.reflect.Field; import org.apache.fory.context.ReadContext; import org.apache.fory.context.WriteContext; import org.apache.fory.exception.ForyException; import org.apache.fory.platform.AndroidSupport; -import org.apache.fory.platform.GraalvmSupport; -import org.apache.fory.reflect.FieldAccessor; +import org.apache.fory.platform.internal._JDKAccess; import org.apache.fory.resolver.TypeResolver; import org.apache.fory.serializer.Serializer; -import org.apache.fory.util.Preconditions; /** * Serializer for scala @@ -37,7 +36,7 @@ @SuppressWarnings("rawtypes") public class SingletonObjectSerializer extends Serializer { private final Field field; - private FieldAccessor accessor; + private MethodHandle accessor; public SingletonObjectSerializer(TypeResolver typeResolver, Class type) { super(typeResolver.getConfig(), type); @@ -70,11 +69,23 @@ public Object read(ReadContext readContext) { throw new ForyException("Failed to read Scala singleton field: " + type, e); } } - FieldAccessor accessor = this.accessor; + MethodHandle accessor = this.accessor; if (accessor == null) { - Preconditions.checkArgument(!GraalvmSupport.isGraalBuildTime()); - accessor = this.accessor = FieldAccessor.createStaticAccessor(field); + accessor = this.accessor = staticGetter(); + } + try { + return accessor.invoke(); + } catch (Throwable e) { + throw new ForyException("Failed to read Scala singleton field: " + type, e); + } + } + + private MethodHandle staticGetter() { + try { + return _JDKAccess._trustedLookup(field.getDeclaringClass()) + .findStaticGetter(field.getDeclaringClass(), field.getName(), field.getType()); + } catch (NoSuchFieldException | IllegalAccessException | RuntimeException e) { + throw new ForyException("Failed to access Scala singleton field: " + type, e); } - return accessor.getObject(null); } } diff --git a/java/fory-core/src/main/java25/org/apache/fory/platform/internal/_Lookup.java b/java/fory-core/src/main/java25/org/apache/fory/platform/internal/_Lookup.java index 66a075b729..2e4fa53562 100644 --- a/java/fory-core/src/main/java25/org/apache/fory/platform/internal/_Lookup.java +++ b/java/fory-core/src/main/java25/org/apache/fory/platform/internal/_Lookup.java @@ -19,6 +19,8 @@ package org.apache.fory.platform.internal; +import java.lang.invoke.MethodHandle; +import java.lang.invoke.MethodType; import java.lang.invoke.MethodHandles.Lookup; import java.lang.reflect.Field; @@ -26,11 +28,18 @@ class _Lookup { // CHECKSTYLE.ON:TypeName private static volatile Lookup implLookup; + private static volatile MethodHandle constructorLookup; // CHECKSTYLE.OFF:MethodName public static Lookup _trustedLookup(Class objectClass) { // CHECKSTYLE.ON:MethodName - return implLookup().in(objectClass); + try { + // IMPL_LOOKUP.in(type) drops trusted modes. JDK26 final-field writes under + // --illegal-final-field-mutation=deny require a target-class /trusted lookup. + return (Lookup) constructorLookup().invoke(objectClass, null, -1); + } catch (Throwable e) { + throw new IllegalStateException(trustedLookupMessage(), e); + } } public static Lookup privateLookupIn(Class targetClass, Lookup caller) { @@ -60,6 +69,30 @@ private static Lookup loadImplLookup() { } } + private static MethodHandle constructorLookup() { + MethodHandle constructor = constructorLookup; + if (constructor == null) { + synchronized (_Lookup.class) { + constructor = constructorLookup; + if (constructor == null) { + constructor = loadConstructorLookup(); + constructorLookup = constructor; + } + } + } + return constructor; + } + + private static MethodHandle loadConstructorLookup() { + try { + return implLookup() + .findConstructor( + Lookup.class, MethodType.methodType(void.class, Class.class, Class.class, int.class)); + } catch (ReflectiveOperationException | RuntimeException e) { + throw new IllegalStateException(trustedLookupMessage(), e); + } + } + private static Lookup implLookup() { Lookup lookup = implLookup; if (lookup == null) { diff --git a/java/fory-core/src/main/java25/org/apache/fory/reflect/FieldAccessorStrategy.java b/java/fory-core/src/main/java25/org/apache/fory/reflect/FieldAccessorStrategy.java index 06bae681f1..4e5241bd1e 100644 --- a/java/fory-core/src/main/java25/org/apache/fory/reflect/FieldAccessorStrategy.java +++ b/java/fory-core/src/main/java25/org/apache/fory/reflect/FieldAccessorStrategy.java @@ -19,12 +19,10 @@ package org.apache.fory.reflect; -import java.lang.invoke.MethodHandles; import java.lang.invoke.VarHandle; import java.lang.reflect.Field; import java.lang.reflect.Modifier; import org.apache.fory.platform.GraalvmSupport; -import org.apache.fory.platform.JdkVersion; import org.apache.fory.platform.internal._JDKAccess; import org.apache.fory.type.TypeUtils; import org.apache.fory.util.Preconditions; @@ -44,11 +42,6 @@ static FieldAccessor createAccessor(Field field) { return createVarHandleAccessor(field); } - static FieldAccessor createStaticAccessor(Field field) { - Preconditions.checkArgument(Modifier.isStatic(field.getModifiers()), field); - return createVarHandleAccessor(field); - } - private static FieldAccessor createVarHandleAccessor(Field field) { if (field.getType() == boolean.class) { return new BooleanAccessor(field); @@ -73,15 +66,9 @@ private static FieldAccessor createVarHandleAccessor(Field field) { private static VarHandle fieldHandle(Field field) { try { - if (canUsePublicField(field)) { - try { - return findFieldHandle(MethodHandles.publicLookup(), field); - } catch (IllegalAccessException ignored) { - // Fall through to the JDK25 trusted lookup. It is enabled by opening - // java.base/java.lang.invoke to Fory, not by opening every target package. - } - } - return findFieldHandle(privateLookup(field), field); + return _JDKAccess + ._trustedLookup(field.getDeclaringClass()) + .findVarHandle(field.getDeclaringClass(), field.getName(), field.getType()); } catch (IllegalAccessException e) { throw accessFailure(field, e); } catch (NoSuchFieldException e) { @@ -89,25 +76,6 @@ private static VarHandle fieldHandle(Field field) { } } - private static VarHandle findFieldHandle(MethodHandles.Lookup lookup, Field field) - throws IllegalAccessException, NoSuchFieldException { - if (Modifier.isStatic(field.getModifiers())) { - return lookup.findStaticVarHandle( - field.getDeclaringClass(), field.getName(), field.getType()); - } - return lookup.findVarHandle(field.getDeclaringClass(), field.getName(), field.getType()); - } - - private static MethodHandles.Lookup privateLookup(Field field) { - Class declaringClass = field.getDeclaringClass(); - return _JDKAccess.privateLookupIn(declaringClass, MethodHandles.lookup()); - } - - private static boolean canUsePublicField(Field field) { - return Modifier.isPublic(field.getModifiers()) - && Modifier.isPublic(field.getDeclaringClass().getModifiers()); - } - private static IllegalStateException accessFailure(Field field, Throwable cause) { return new IllegalStateException( "Cannot access field " @@ -117,153 +85,16 @@ private static IllegalStateException accessFailure(Field field, Throwable cause) cause); } - private static UnsupportedOperationException unsupportedWrite(Field field, Throwable cause) { - return new UnsupportedOperationException( - "Field cannot be written through supported JDK access APIs: " + field, cause); - } - - private static IllegalStateException finalMutationFailure(Field field, Throwable cause) { - String versionMessage = - JdkVersion.MAJOR_VERSION >= 26 - ? "On JDK26+, start the JVM with " - + "--enable-final-field-mutation=org.apache.fory.core. " - : ""; - return new IllegalStateException( - "Cannot write final field " - + field - + ". " - + versionMessage - + "The declaring package must be open to org.apache.fory.core when it is in a " - + "named module.", - cause); - } - private static RuntimeException accessorFailure(Field field, Throwable cause) { return new RuntimeException("Failed to access field: " + field, cause); } private abstract static class InstanceAccessor extends FieldAccessor { protected final VarHandle handle; - protected final boolean isStatic; - protected final boolean isFinal; - protected volatile Field finalField; InstanceAccessor(Field field) { super(field); handle = fieldHandle(field); - isStatic = Modifier.isStatic(field.getModifiers()); - isFinal = Modifier.isFinal(field.getModifiers()); - } - - private static Field createFinalField(Field field) { - try { - field.setAccessible(true); - return field; - } catch (RuntimeException e) { - throw finalMutationFailure(field, e); - } - } - - private Field finalField(Throwable cause) { - if (isStatic || !isFinal) { - throw unsupportedWrite(field, cause); - } - Field setter = finalField; - if (setter == null) { - setter = createFinalField(field); - finalField = setter; - } - return setter; - } - - protected void setFinal(Object obj, Object value, Throwable cause) { - Field setter = finalField(cause); - checkObj(obj); - try { - setter.set(obj, value); - } catch (IllegalAccessException | IllegalArgumentException e) { - throw finalMutationFailure(field, e); - } - } - - protected void setFinalBoolean(Object obj, boolean value, Throwable cause) { - Field setter = finalField(cause); - checkObj(obj); - try { - setter.setBoolean(obj, value); - } catch (IllegalAccessException | IllegalArgumentException e) { - throw finalMutationFailure(field, e); - } - } - - protected void setFinalByte(Object obj, byte value, Throwable cause) { - Field setter = finalField(cause); - checkObj(obj); - try { - setter.setByte(obj, value); - } catch (IllegalAccessException | IllegalArgumentException e) { - throw finalMutationFailure(field, e); - } - } - - protected void setFinalChar(Object obj, char value, Throwable cause) { - Field setter = finalField(cause); - checkObj(obj); - try { - setter.setChar(obj, value); - } catch (IllegalAccessException | IllegalArgumentException e) { - throw finalMutationFailure(field, e); - } - } - - protected void setFinalShort(Object obj, short value, Throwable cause) { - Field setter = finalField(cause); - checkObj(obj); - try { - setter.setShort(obj, value); - } catch (IllegalAccessException | IllegalArgumentException e) { - throw finalMutationFailure(field, e); - } - } - - protected void setFinalInt(Object obj, int value, Throwable cause) { - Field setter = finalField(cause); - checkObj(obj); - try { - setter.setInt(obj, value); - } catch (IllegalAccessException | IllegalArgumentException e) { - throw finalMutationFailure(field, e); - } - } - - protected void setFinalLong(Object obj, long value, Throwable cause) { - Field setter = finalField(cause); - checkObj(obj); - try { - setter.setLong(obj, value); - } catch (IllegalAccessException | IllegalArgumentException e) { - throw finalMutationFailure(field, e); - } - } - - protected void setFinalFloat(Object obj, float value, Throwable cause) { - Field setter = finalField(cause); - checkObj(obj); - try { - setter.setFloat(obj, value); - } catch (IllegalAccessException | IllegalArgumentException e) { - throw finalMutationFailure(field, e); - } - } - - protected void setFinalDouble(Object obj, double value, Throwable cause) { - Field setter = finalField(cause); - checkObj(obj); - try { - setter.setDouble(obj, value); - } catch (IllegalAccessException | IllegalArgumentException e) { - throw finalMutationFailure(field, e); - } } } @@ -281,10 +112,6 @@ public Object get(Object obj) { @Override public boolean getBoolean(Object obj) { - if (isStatic) { - return (boolean) handle.get(); - } - checkObj(obj); return (boolean) handle.get(obj); } @@ -295,19 +122,10 @@ public void set(Object obj, Object value) { @Override public void putBoolean(Object obj, boolean value) { - if (isFinal) { - setFinalBoolean(obj, value, null); - return; - } try { - if (isStatic) { - handle.set(value); - } else { - checkObj(obj); - handle.set(obj, value); - } - } catch (UnsupportedOperationException e) { - setFinalBoolean(obj, value, e); + handle.set(obj, value); + } catch (RuntimeException e) { + throw accessorFailure(field, e); } } } @@ -326,10 +144,6 @@ public Byte get(Object obj) { @Override public byte getByte(Object obj) { - if (isStatic) { - return (byte) handle.get(); - } - checkObj(obj); return (byte) handle.get(obj); } @@ -340,19 +154,10 @@ public void set(Object obj, Object value) { @Override public void putByte(Object obj, byte value) { - if (isFinal) { - setFinalByte(obj, value, null); - return; - } try { - if (isStatic) { - handle.set(value); - } else { - checkObj(obj); - handle.set(obj, value); - } - } catch (UnsupportedOperationException e) { - setFinalByte(obj, value, e); + handle.set(obj, value); + } catch (RuntimeException e) { + throw accessorFailure(field, e); } } } @@ -371,10 +176,6 @@ public Character get(Object obj) { @Override public char getChar(Object obj) { - if (isStatic) { - return (char) handle.get(); - } - checkObj(obj); return (char) handle.get(obj); } @@ -385,19 +186,10 @@ public void set(Object obj, Object value) { @Override public void putChar(Object obj, char value) { - if (isFinal) { - setFinalChar(obj, value, null); - return; - } try { - if (isStatic) { - handle.set(value); - } else { - checkObj(obj); - handle.set(obj, value); - } - } catch (UnsupportedOperationException e) { - setFinalChar(obj, value, e); + handle.set(obj, value); + } catch (RuntimeException e) { + throw accessorFailure(field, e); } } } @@ -416,10 +208,6 @@ public Short get(Object obj) { @Override public short getShort(Object obj) { - if (isStatic) { - return (short) handle.get(); - } - checkObj(obj); return (short) handle.get(obj); } @@ -430,19 +218,10 @@ public void set(Object obj, Object value) { @Override public void putShort(Object obj, short value) { - if (isFinal) { - setFinalShort(obj, value, null); - return; - } try { - if (isStatic) { - handle.set(value); - } else { - checkObj(obj); - handle.set(obj, value); - } - } catch (UnsupportedOperationException e) { - setFinalShort(obj, value, e); + handle.set(obj, value); + } catch (RuntimeException e) { + throw accessorFailure(field, e); } } } @@ -461,10 +240,6 @@ public Integer get(Object obj) { @Override public int getInt(Object obj) { - if (isStatic) { - return (int) handle.get(); - } - checkObj(obj); return (int) handle.get(obj); } @@ -475,19 +250,10 @@ public void set(Object obj, Object value) { @Override public void putInt(Object obj, int value) { - if (isFinal) { - setFinalInt(obj, value, null); - return; - } try { - if (isStatic) { - handle.set(value); - } else { - checkObj(obj); - handle.set(obj, value); - } - } catch (UnsupportedOperationException e) { - setFinalInt(obj, value, e); + handle.set(obj, value); + } catch (RuntimeException e) { + throw accessorFailure(field, e); } } } @@ -506,10 +272,6 @@ public Long get(Object obj) { @Override public long getLong(Object obj) { - if (isStatic) { - return (long) handle.get(); - } - checkObj(obj); return (long) handle.get(obj); } @@ -520,19 +282,10 @@ public void set(Object obj, Object value) { @Override public void putLong(Object obj, long value) { - if (isFinal) { - setFinalLong(obj, value, null); - return; - } try { - if (isStatic) { - handle.set(value); - } else { - checkObj(obj); - handle.set(obj, value); - } - } catch (UnsupportedOperationException e) { - setFinalLong(obj, value, e); + handle.set(obj, value); + } catch (RuntimeException e) { + throw accessorFailure(field, e); } } } @@ -551,10 +304,6 @@ public Object get(Object obj) { @Override public float getFloat(Object obj) { - if (isStatic) { - return (float) handle.get(); - } - checkObj(obj); return (float) handle.get(obj); } @@ -565,19 +314,10 @@ public void set(Object obj, Object value) { @Override public void putFloat(Object obj, float value) { - if (isFinal) { - setFinalFloat(obj, value, null); - return; - } try { - if (isStatic) { - handle.set(value); - } else { - checkObj(obj); - handle.set(obj, value); - } - } catch (UnsupportedOperationException e) { - setFinalFloat(obj, value, e); + handle.set(obj, value); + } catch (RuntimeException e) { + throw accessorFailure(field, e); } } } @@ -596,10 +336,6 @@ public Object get(Object obj) { @Override public double getDouble(Object obj) { - if (isStatic) { - return (double) handle.get(); - } - checkObj(obj); return (double) handle.get(obj); } @@ -610,19 +346,10 @@ public void set(Object obj, Object value) { @Override public void putDouble(Object obj, double value) { - if (isFinal) { - setFinalDouble(obj, value, null); - return; - } try { - if (isStatic) { - handle.set(value); - } else { - checkObj(obj); - handle.set(obj, value); - } - } catch (UnsupportedOperationException e) { - setFinalDouble(obj, value, e); + handle.set(obj, value); + } catch (RuntimeException e) { + throw accessorFailure(field, e); } } } @@ -636,39 +363,19 @@ public ObjectAccessor(Field field) { @Override public Object get(Object obj) { - if (isStatic) { - return handle.get(); - } - checkObj(obj); return handle.get(obj); } @Override public void set(Object obj, Object value) { - if (isFinal) { - setFinal(obj, value, null); - return; - } try { - if (isStatic) { - handle.set(value); - } else { - checkObj(obj); - handle.set(obj, value); - } - } catch (UnsupportedOperationException e) { - setFinal(obj, value, e); + handle.set(obj, value); + } catch (RuntimeException e) { + throw accessorFailure(field, e); } } } - static final class StaticObjectAccessor extends ObjectAccessor { - StaticObjectAccessor(Field field) { - super(field); - Preconditions.checkArgument(Modifier.isStatic(field.getModifiers()), field); - } - } - static final class GeneratedAccessor extends InstanceAccessor { GeneratedAccessor(Field field) { super(field); @@ -677,10 +384,6 @@ static final class GeneratedAccessor extends InstanceAccessor { @Override public Object get(Object obj) { try { - if (isStatic) { - return handle.get(); - } - checkObj(obj); return handle.get(obj); } catch (RuntimeException e) { throw accessorFailure(field, e); @@ -689,19 +392,8 @@ public Object get(Object obj) { @Override public void set(Object obj, Object value) { - if (isFinal) { - setFinal(obj, value, null); - return; - } try { - if (isStatic) { - handle.set(value); - } else { - checkObj(obj); - handle.set(obj, value); - } - } catch (UnsupportedOperationException e) { - setFinal(obj, value, e); + handle.set(obj, value); } catch (RuntimeException e) { throw accessorFailure(field, e); } @@ -710,10 +402,6 @@ public void set(Object obj, Object value) { @Override public boolean getBoolean(Object obj) { try { - if (isStatic) { - return (boolean) handle.get(); - } - checkObj(obj); return (boolean) handle.get(obj); } catch (RuntimeException e) { throw accessorFailure(field, e); @@ -722,19 +410,8 @@ public boolean getBoolean(Object obj) { @Override public void putBoolean(Object obj, boolean value) { - if (isFinal) { - setFinalBoolean(obj, value, null); - return; - } try { - if (isStatic) { - handle.set(value); - } else { - checkObj(obj); - handle.set(obj, value); - } - } catch (UnsupportedOperationException e) { - setFinalBoolean(obj, value, e); + handle.set(obj, value); } catch (RuntimeException e) { throw accessorFailure(field, e); } @@ -743,10 +420,6 @@ public void putBoolean(Object obj, boolean value) { @Override public byte getByte(Object obj) { try { - if (isStatic) { - return (byte) handle.get(); - } - checkObj(obj); return (byte) handle.get(obj); } catch (RuntimeException e) { throw accessorFailure(field, e); @@ -755,19 +428,8 @@ public byte getByte(Object obj) { @Override public void putByte(Object obj, byte value) { - if (isFinal) { - setFinalByte(obj, value, null); - return; - } try { - if (isStatic) { - handle.set(value); - } else { - checkObj(obj); - handle.set(obj, value); - } - } catch (UnsupportedOperationException e) { - setFinalByte(obj, value, e); + handle.set(obj, value); } catch (RuntimeException e) { throw accessorFailure(field, e); } @@ -776,10 +438,6 @@ public void putByte(Object obj, byte value) { @Override public char getChar(Object obj) { try { - if (isStatic) { - return (char) handle.get(); - } - checkObj(obj); return (char) handle.get(obj); } catch (RuntimeException e) { throw accessorFailure(field, e); @@ -788,19 +446,8 @@ public char getChar(Object obj) { @Override public void putChar(Object obj, char value) { - if (isFinal) { - setFinalChar(obj, value, null); - return; - } try { - if (isStatic) { - handle.set(value); - } else { - checkObj(obj); - handle.set(obj, value); - } - } catch (UnsupportedOperationException e) { - setFinalChar(obj, value, e); + handle.set(obj, value); } catch (RuntimeException e) { throw accessorFailure(field, e); } @@ -809,10 +456,6 @@ public void putChar(Object obj, char value) { @Override public short getShort(Object obj) { try { - if (isStatic) { - return (short) handle.get(); - } - checkObj(obj); return (short) handle.get(obj); } catch (RuntimeException e) { throw accessorFailure(field, e); @@ -821,19 +464,8 @@ public short getShort(Object obj) { @Override public void putShort(Object obj, short value) { - if (isFinal) { - setFinalShort(obj, value, null); - return; - } try { - if (isStatic) { - handle.set(value); - } else { - checkObj(obj); - handle.set(obj, value); - } - } catch (UnsupportedOperationException e) { - setFinalShort(obj, value, e); + handle.set(obj, value); } catch (RuntimeException e) { throw accessorFailure(field, e); } @@ -842,10 +474,6 @@ public void putShort(Object obj, short value) { @Override public int getInt(Object obj) { try { - if (isStatic) { - return (int) handle.get(); - } - checkObj(obj); return (int) handle.get(obj); } catch (RuntimeException e) { throw accessorFailure(field, e); @@ -854,19 +482,8 @@ public int getInt(Object obj) { @Override public void putInt(Object obj, int value) { - if (isFinal) { - setFinalInt(obj, value, null); - return; - } try { - if (isStatic) { - handle.set(value); - } else { - checkObj(obj); - handle.set(obj, value); - } - } catch (UnsupportedOperationException e) { - setFinalInt(obj, value, e); + handle.set(obj, value); } catch (RuntimeException e) { throw accessorFailure(field, e); } @@ -875,10 +492,6 @@ public void putInt(Object obj, int value) { @Override public long getLong(Object obj) { try { - if (isStatic) { - return (long) handle.get(); - } - checkObj(obj); return (long) handle.get(obj); } catch (RuntimeException e) { throw accessorFailure(field, e); @@ -887,19 +500,8 @@ public long getLong(Object obj) { @Override public void putLong(Object obj, long value) { - if (isFinal) { - setFinalLong(obj, value, null); - return; - } try { - if (isStatic) { - handle.set(value); - } else { - checkObj(obj); - handle.set(obj, value); - } - } catch (UnsupportedOperationException e) { - setFinalLong(obj, value, e); + handle.set(obj, value); } catch (RuntimeException e) { throw accessorFailure(field, e); } @@ -908,10 +510,6 @@ public void putLong(Object obj, long value) { @Override public float getFloat(Object obj) { try { - if (isStatic) { - return (float) handle.get(); - } - checkObj(obj); return (float) handle.get(obj); } catch (RuntimeException e) { throw accessorFailure(field, e); @@ -920,19 +518,8 @@ public float getFloat(Object obj) { @Override public void putFloat(Object obj, float value) { - if (isFinal) { - setFinalFloat(obj, value, null); - return; - } try { - if (isStatic) { - handle.set(value); - } else { - checkObj(obj); - handle.set(obj, value); - } - } catch (UnsupportedOperationException e) { - setFinalFloat(obj, value, e); + handle.set(obj, value); } catch (RuntimeException e) { throw accessorFailure(field, e); } @@ -941,10 +528,6 @@ public void putFloat(Object obj, float value) { @Override public double getDouble(Object obj) { try { - if (isStatic) { - return (double) handle.get(); - } - checkObj(obj); return (double) handle.get(obj); } catch (RuntimeException e) { throw accessorFailure(field, e); @@ -953,19 +536,8 @@ public double getDouble(Object obj) { @Override public void putDouble(Object obj, double value) { - if (isFinal) { - setFinalDouble(obj, value, null); - return; - } try { - if (isStatic) { - handle.set(value); - } else { - checkObj(obj); - handle.set(obj, value); - } - } catch (UnsupportedOperationException e) { - setFinalDouble(obj, value, e); + handle.set(obj, value); } catch (RuntimeException e) { throw accessorFailure(field, e); } diff --git a/java/fory-core/src/test/java/org/apache/fory/TestUtils.java b/java/fory-core/src/test/java/org/apache/fory/TestUtils.java index 746e640004..321ac38f71 100644 --- a/java/fory-core/src/test/java/org/apache/fory/TestUtils.java +++ b/java/fory-core/src/test/java/org/apache/fory/TestUtils.java @@ -85,7 +85,7 @@ private static List forkJvmArgs() { if (hasInputArg("--sun-misc-unsafe-memory-access=deny")) { args.add("--sun-misc-unsafe-memory-access=deny"); } - finalFieldMutationArg().ifPresent(args::add); + finalFieldPolicyArg().ifPresent(args::add); } return args; } @@ -108,9 +108,9 @@ private static boolean hasInputArg(String arg) { return ManagementFactory.getRuntimeMXBean().getInputArguments().contains(arg); } - private static Optional finalFieldMutationArg() { + private static Optional finalFieldPolicyArg() { return ManagementFactory.getRuntimeMXBean().getInputArguments().stream() - .filter(arg -> arg.startsWith("--enable-final-field-mutation=")) + .filter(arg -> arg.startsWith("--illegal-final-field-mutation=")) .findFirst(); } From 18a089013024b1f0137cadffa63c33948bfb74eb Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Tue, 2 Jun 2026 17:05:44 +0800 Subject: [PATCH 50/69] refactor(java): collapse field accessor strategies --- .agents/languages/java.md | 11 +- ci/run_ci.sh | 8 - ci/tasks/java.py | 9 +- .../fory/reflect/FieldAccessorStrategy.java | 351 ++++-------- .../fory/reflect/FieldAccessorStrategy.java | 534 +++++------------- 5 files changed, 254 insertions(+), 659 deletions(-) diff --git a/.agents/languages/java.md b/.agents/languages/java.md index 8c31c89fbb..44614889ff 100644 --- a/.agents/languages/java.md +++ b/.agents/languages/java.md @@ -109,9 +109,14 @@ Load this file when changing anything under `java/` or when Java drives a cross- - JDK25+ final-instance field mutation should use the same trusted-lookup `VarHandle` owner path as non-final field mutation. Do not add a final-field-only `MethodHandle` setter path or ordinary reflective `Field.set*` fallback. -- Do not call `FieldAccessor.checkObj` in the JDK25+ VarHandle field-access hot path. VarHandle - validates null and receiver type itself, while root Unsafe offset access still needs explicit - receiver validation because Unsafe does not. +- JDK25+ `FieldAccessorStrategy` should use one final trusted-lookup `VarHandle` instance accessor + with dense access-kind switches, not public primitive/object accessor classes or a JDK25 + `GeneratedAccessor`. Do not wrap `VarHandle.get/set` in hot-path try/catch blocks and do not call + `FieldAccessor.checkObj`; VarHandle validates null and receiver type itself. Root Unsafe offset + access still needs explicit receiver validation because Unsafe does not. +- Keep JDK26 `--illegal-final-field-mutation=deny` scoped to the JPMS runtime tests that prove + Fory's field-access path. Do not put it in global Maven `JDK_JAVA_OPTIONS`, because build tools + such as Lombok may perform their own reflective final-field access during compilation. - JDK25+ collection serializers must fail unsupported `Collections.newSetFromMap` backing maps before writing or copying. Do not rewrite them to `HashMap`, because that changes equality semantics and can drop entries. diff --git a/ci/run_ci.sh b/ci/run_ci.sh index 2abdf7157d..9edee67013 100755 --- a/ci/run_ci.sh +++ b/ci/run_ci.sh @@ -107,17 +107,9 @@ jdk25_access_options() { printf " %s" "--add-opens=java.base/java.lang.invoke=${fory_open_targets}" } -jdk26_final_field_policy_options() { - printf "%s" "--illegal-final-field-mutation=deny" -} - jdk25_plus_options() { - local java_major="$1" local fory_targets="${2:-org.apache.fory.core}" printf "%s" "$(jdk25_access_options "$fory_targets")" - if [[ "$java_major" -ge 26 ]]; then - printf " %s" "$(jdk26_final_field_policy_options)" - fi } jdk25_javac_options() { diff --git a/ci/tasks/java.py b/ci/tasks/java.py index 022abc4241..5c8ee50d14 100644 --- a/ci/tasks/java.py +++ b/ci/tasks/java.py @@ -84,15 +84,8 @@ def jdk25_access_options(fory_targets="org.apache.fory.core"): ] -def jdk26_final_field_options(fory_targets="org.apache.fory.core"): - return [f"--enable-final-field-mutation={fory_targets}"] - - def jdk25_plus_options(java_version, fory_targets="org.apache.fory.core"): - options = jdk25_access_options(fory_targets) - if int(java_version) >= 26: - options.extend(jdk26_final_field_options(fory_targets)) - return options + return jdk25_access_options(fory_targets) def jdk25_javac_options(): diff --git a/java/fory-core/src/main/java/org/apache/fory/reflect/FieldAccessorStrategy.java b/java/fory-core/src/main/java/org/apache/fory/reflect/FieldAccessorStrategy.java index cca430f393..2ab34a3625 100644 --- a/java/fory-core/src/main/java/org/apache/fory/reflect/FieldAccessorStrategy.java +++ b/java/fory-core/src/main/java/org/apache/fory/reflect/FieldAccessorStrategy.java @@ -31,7 +31,6 @@ import org.apache.fory.platform.GraalvmSupport; import org.apache.fory.platform.internal._JDKAccess; import org.apache.fory.platform.internal._UnsafeUtils; -import org.apache.fory.type.TypeUtils; import org.apache.fory.util.Preconditions; import sun.misc.Unsafe; @@ -50,6 +49,14 @@ final class FieldAccessorStrategy { private FieldAccessorStrategy() {} + static FieldAccessor createAccessor(Field field) { + Preconditions.checkArgument(!Modifier.isStatic(field.getModifiers()), field); + if (GraalvmSupport.isGraalBuildTime()) { + return new GeneratedAccessor(field); + } + return new InstanceAccessor(field); + } + private static long fieldOffset(Field field) { if (AndroidSupport.IS_ANDROID) { return -1; @@ -83,60 +90,8 @@ private static int accessKind(Field field) { return OBJECT_ACCESS; } - private static void copyField( - long fieldOffset, - int accessKind, - Object sourceObject, - Object targetObject, - FieldAccessor accessor) { - switch (accessKind) { - case BOOLEAN_ACCESS: - UNSAFE.putBoolean(targetObject, fieldOffset, UNSAFE.getBoolean(sourceObject, fieldOffset)); - return; - case BYTE_ACCESS: - UNSAFE.putByte(targetObject, fieldOffset, UNSAFE.getByte(sourceObject, fieldOffset)); - return; - case CHAR_ACCESS: - UNSAFE.putChar(targetObject, fieldOffset, UNSAFE.getChar(sourceObject, fieldOffset)); - return; - case SHORT_ACCESS: - UNSAFE.putShort(targetObject, fieldOffset, UNSAFE.getShort(sourceObject, fieldOffset)); - return; - case INT_ACCESS: - UNSAFE.putInt(targetObject, fieldOffset, UNSAFE.getInt(sourceObject, fieldOffset)); - return; - case LONG_ACCESS: - UNSAFE.putLong(targetObject, fieldOffset, UNSAFE.getLong(sourceObject, fieldOffset)); - return; - case FLOAT_ACCESS: - UNSAFE.putFloat(targetObject, fieldOffset, UNSAFE.getFloat(sourceObject, fieldOffset)); - return; - case DOUBLE_ACCESS: - UNSAFE.putDouble(targetObject, fieldOffset, UNSAFE.getDouble(sourceObject, fieldOffset)); - return; - case OBJECT_ACCESS: - UNSAFE.putObject(targetObject, fieldOffset, UNSAFE.getObject(sourceObject, fieldOffset)); - return; - default: - accessor.putObject(targetObject, accessor.getObject(sourceObject)); - } - } - - private static void copyObjectField( - long fieldOffset, - int accessKind, - Object sourceObject, - Object targetObject, - FieldAccessor accessor) { - if (accessKind == OBJECT_ACCESS) { - UNSAFE.putObject(targetObject, fieldOffset, UNSAFE.getObject(sourceObject, fieldOffset)); - } else { - accessor.putObject(targetObject, accessor.getObject(sourceObject)); - } - } - - private abstract static class InstanceAccessor extends FieldAccessor { - protected final long fieldOffset; + static final class InstanceAccessor extends FieldAccessor { + private final long fieldOffset; private final int accessKind; InstanceAccessor(Field field) { @@ -146,52 +101,111 @@ private abstract static class InstanceAccessor extends FieldAccessor { } @Override - public void copy(Object sourceObject, Object targetObject) { - copyField(fieldOffset, accessKind, sourceObject, targetObject, this); + public Object get(Object obj) { + checkObj(obj); + switch (accessKind) { + case BOOLEAN_ACCESS: + return UNSAFE.getBoolean(obj, fieldOffset); + case BYTE_ACCESS: + return UNSAFE.getByte(obj, fieldOffset); + case CHAR_ACCESS: + return UNSAFE.getChar(obj, fieldOffset); + case SHORT_ACCESS: + return UNSAFE.getShort(obj, fieldOffset); + case INT_ACCESS: + return UNSAFE.getInt(obj, fieldOffset); + case LONG_ACCESS: + return UNSAFE.getLong(obj, fieldOffset); + case FLOAT_ACCESS: + return UNSAFE.getFloat(obj, fieldOffset); + case DOUBLE_ACCESS: + return UNSAFE.getDouble(obj, fieldOffset); + case OBJECT_ACCESS: + return UNSAFE.getObject(obj, fieldOffset); + default: + throw new IllegalStateException("Unsupported access kind " + accessKind); + } } @Override - public void copyObject(Object sourceObject, Object targetObject) { - copyObjectField(fieldOffset, accessKind, sourceObject, targetObject, this); - } - } - - static FieldAccessor createAccessor(Field field) { - Preconditions.checkArgument(!Modifier.isStatic(field.getModifiers()), field); - if (GraalvmSupport.isGraalBuildTime()) { - return new GeneratedAccessor(field); - } - if (field.getType() == boolean.class) { - return new BooleanAccessor(field); - } else if (field.getType() == byte.class) { - return new ByteAccessor(field); - } else if (field.getType() == char.class) { - return new CharAccessor(field); - } else if (field.getType() == short.class) { - return new ShortAccessor(field); - } else if (field.getType() == int.class) { - return new IntAccessor(field); - } else if (field.getType() == long.class) { - return new LongAccessor(field); - } else if (field.getType() == float.class) { - return new FloatAccessor(field); - } else if (field.getType() == double.class) { - return new DoubleAccessor(field); - } else { - return new ObjectAccessor(field); + public void set(Object obj, Object value) { + checkObj(obj); + switch (accessKind) { + case BOOLEAN_ACCESS: + UNSAFE.putBoolean(obj, fieldOffset, (Boolean) value); + return; + case BYTE_ACCESS: + UNSAFE.putByte(obj, fieldOffset, (Byte) value); + return; + case CHAR_ACCESS: + UNSAFE.putChar(obj, fieldOffset, (Character) value); + return; + case SHORT_ACCESS: + UNSAFE.putShort(obj, fieldOffset, (Short) value); + return; + case INT_ACCESS: + UNSAFE.putInt(obj, fieldOffset, (Integer) value); + return; + case LONG_ACCESS: + UNSAFE.putLong(obj, fieldOffset, (Long) value); + return; + case FLOAT_ACCESS: + UNSAFE.putFloat(obj, fieldOffset, (Float) value); + return; + case DOUBLE_ACCESS: + UNSAFE.putDouble(obj, fieldOffset, (Double) value); + return; + case OBJECT_ACCESS: + UNSAFE.putObject(obj, fieldOffset, value); + return; + default: + throw new IllegalStateException("Unsupported access kind " + accessKind); + } } - } - /** Primitive boolean accessor. */ - public static class BooleanAccessor extends InstanceAccessor { - public BooleanAccessor(Field field) { - super(field); - Preconditions.checkArgument(field.getType() == boolean.class); + @Override + public void copy(Object sourceObject, Object targetObject) { + switch (accessKind) { + case BOOLEAN_ACCESS: + UNSAFE.putBoolean( + targetObject, fieldOffset, UNSAFE.getBoolean(sourceObject, fieldOffset)); + return; + case BYTE_ACCESS: + UNSAFE.putByte(targetObject, fieldOffset, UNSAFE.getByte(sourceObject, fieldOffset)); + return; + case CHAR_ACCESS: + UNSAFE.putChar(targetObject, fieldOffset, UNSAFE.getChar(sourceObject, fieldOffset)); + return; + case SHORT_ACCESS: + UNSAFE.putShort(targetObject, fieldOffset, UNSAFE.getShort(sourceObject, fieldOffset)); + return; + case INT_ACCESS: + UNSAFE.putInt(targetObject, fieldOffset, UNSAFE.getInt(sourceObject, fieldOffset)); + return; + case LONG_ACCESS: + UNSAFE.putLong(targetObject, fieldOffset, UNSAFE.getLong(sourceObject, fieldOffset)); + return; + case FLOAT_ACCESS: + UNSAFE.putFloat(targetObject, fieldOffset, UNSAFE.getFloat(sourceObject, fieldOffset)); + return; + case DOUBLE_ACCESS: + UNSAFE.putDouble(targetObject, fieldOffset, UNSAFE.getDouble(sourceObject, fieldOffset)); + return; + case OBJECT_ACCESS: + UNSAFE.putObject(targetObject, fieldOffset, UNSAFE.getObject(sourceObject, fieldOffset)); + return; + default: + super.copy(sourceObject, targetObject); + } } @Override - public Object get(Object obj) { - return getBoolean(obj); + public void copyObject(Object sourceObject, Object targetObject) { + if (accessKind == OBJECT_ACCESS) { + UNSAFE.putObject(targetObject, fieldOffset, UNSAFE.getObject(sourceObject, fieldOffset)); + } else { + super.copyObject(sourceObject, targetObject); + } } @Override @@ -200,29 +214,11 @@ public boolean getBoolean(Object obj) { return UNSAFE.getBoolean(obj, fieldOffset); } - @Override - public void set(Object obj, Object value) { - putBoolean(obj, (Boolean) value); - } - @Override public void putBoolean(Object obj, boolean value) { checkObj(obj); UNSAFE.putBoolean(obj, fieldOffset, value); } - } - - /** Primitive byte accessor. */ - public static class ByteAccessor extends InstanceAccessor { - public ByteAccessor(Field field) { - super(field); - Preconditions.checkArgument(field.getType() == byte.class); - } - - @Override - public Byte get(Object obj) { - return getByte(obj); - } @Override public byte getByte(Object obj) { @@ -230,29 +226,11 @@ public byte getByte(Object obj) { return UNSAFE.getByte(obj, fieldOffset); } - @Override - public void set(Object obj, Object value) { - putByte(obj, (Byte) value); - } - @Override public void putByte(Object obj, byte value) { checkObj(obj); UNSAFE.putByte(obj, fieldOffset, value); } - } - - /** Primitive char accessor. */ - public static class CharAccessor extends InstanceAccessor { - public CharAccessor(Field field) { - super(field); - Preconditions.checkArgument(field.getType() == char.class); - } - - @Override - public Character get(Object obj) { - return getChar(obj); - } @Override public char getChar(Object obj) { @@ -260,29 +238,11 @@ public char getChar(Object obj) { return UNSAFE.getChar(obj, fieldOffset); } - @Override - public void set(Object obj, Object value) { - putChar(obj, (Character) value); - } - @Override public void putChar(Object obj, char value) { checkObj(obj); UNSAFE.putChar(obj, fieldOffset, value); } - } - - /** Primitive short accessor. */ - public static class ShortAccessor extends InstanceAccessor { - public ShortAccessor(Field field) { - super(field); - Preconditions.checkArgument(field.getType() == short.class); - } - - @Override - public Short get(Object obj) { - return getShort(obj); - } @Override public short getShort(Object obj) { @@ -290,29 +250,11 @@ public short getShort(Object obj) { return UNSAFE.getShort(obj, fieldOffset); } - @Override - public void set(Object obj, Object value) { - putShort(obj, (Short) value); - } - @Override public void putShort(Object obj, short value) { checkObj(obj); UNSAFE.putShort(obj, fieldOffset, value); } - } - - /** Primitive int accessor. */ - public static class IntAccessor extends InstanceAccessor { - public IntAccessor(Field field) { - super(field); - Preconditions.checkArgument(field.getType() == int.class); - } - - @Override - public Integer get(Object obj) { - return getInt(obj); - } @Override public int getInt(Object obj) { @@ -320,29 +262,11 @@ public int getInt(Object obj) { return UNSAFE.getInt(obj, fieldOffset); } - @Override - public void set(Object obj, Object value) { - putInt(obj, (Integer) value); - } - @Override public void putInt(Object obj, int value) { checkObj(obj); UNSAFE.putInt(obj, fieldOffset, value); } - } - - /** Primitive long accessor. */ - public static class LongAccessor extends InstanceAccessor { - public LongAccessor(Field field) { - super(field); - Preconditions.checkArgument(field.getType() == long.class); - } - - @Override - public Long get(Object obj) { - return getLong(obj); - } @Override public long getLong(Object obj) { @@ -350,29 +274,11 @@ public long getLong(Object obj) { return UNSAFE.getLong(obj, fieldOffset); } - @Override - public void set(Object obj, Object value) { - putLong(obj, (Long) value); - } - @Override public void putLong(Object obj, long value) { checkObj(obj); UNSAFE.putLong(obj, fieldOffset, value); } - } - - /** Primitive float accessor. */ - public static class FloatAccessor extends InstanceAccessor { - public FloatAccessor(Field field) { - super(field); - Preconditions.checkArgument(field.getType() == float.class); - } - - @Override - public Object get(Object obj) { - return getFloat(obj); - } @Override public float getFloat(Object obj) { @@ -380,29 +286,11 @@ public float getFloat(Object obj) { return UNSAFE.getFloat(obj, fieldOffset); } - @Override - public void set(Object obj, Object value) { - putFloat(obj, (Float) value); - } - @Override public void putFloat(Object obj, float value) { checkObj(obj); UNSAFE.putFloat(obj, fieldOffset, value); } - } - - /** Primitive double accessor. */ - public static class DoubleAccessor extends InstanceAccessor { - public DoubleAccessor(Field field) { - super(field); - Preconditions.checkArgument(field.getType() == double.class); - } - - @Override - public Object get(Object obj) { - return getDouble(obj); - } @Override public double getDouble(Object obj) { @@ -410,11 +298,6 @@ public double getDouble(Object obj) { return UNSAFE.getDouble(obj, fieldOffset); } - @Override - public void set(Object obj, Object value) { - putDouble(obj, (Double) value); - } - @Override public void putDouble(Object obj, double value) { checkObj(obj); @@ -422,26 +305,6 @@ public void putDouble(Object obj, double value) { } } - /** Object accessor. */ - public static class ObjectAccessor extends InstanceAccessor { - public ObjectAccessor(Field field) { - super(field); - Preconditions.checkArgument(!TypeUtils.isPrimitive(field.getType())); - } - - @Override - public Object get(Object obj) { - checkObj(obj); - return UNSAFE.getObject(obj, fieldOffset); - } - - @Override - public void set(Object obj, Object value) { - checkObj(obj); - UNSAFE.putObject(obj, fieldOffset, value); - } - } - static final class GeneratedAccessor extends FieldAccessor { private static final ClassValueCache>> cache = ClassValueCache.newClassKeyCache(8); diff --git a/java/fory-core/src/main/java25/org/apache/fory/reflect/FieldAccessorStrategy.java b/java/fory-core/src/main/java25/org/apache/fory/reflect/FieldAccessorStrategy.java index 4e5241bd1e..99695c07da 100644 --- a/java/fory-core/src/main/java25/org/apache/fory/reflect/FieldAccessorStrategy.java +++ b/java/fory-core/src/main/java25/org/apache/fory/reflect/FieldAccessorStrategy.java @@ -22,46 +22,51 @@ import java.lang.invoke.VarHandle; import java.lang.reflect.Field; import java.lang.reflect.Modifier; -import org.apache.fory.platform.GraalvmSupport; import org.apache.fory.platform.internal._JDKAccess; -import org.apache.fory.type.TypeUtils; import org.apache.fory.util.Preconditions; final class FieldAccessorStrategy { + private static final int BOOLEAN_ACCESS = 1; + private static final int BYTE_ACCESS = 2; + private static final int CHAR_ACCESS = 3; + private static final int SHORT_ACCESS = 4; + private static final int INT_ACCESS = 5; + private static final int LONG_ACCESS = 6; + private static final int FLOAT_ACCESS = 7; + private static final int DOUBLE_ACCESS = 8; + private static final int OBJECT_ACCESS = 9; + private FieldAccessorStrategy() {} static FieldAccessor createAccessor(Field field) { Preconditions.checkArgument(!Modifier.isStatic(field.getModifiers()), field); - if (GraalvmSupport.isGraalBuildTime() || GraalvmSupport.IN_GRAALVM_NATIVE_IMAGE) { - return new GeneratedAccessor(field); - } FieldAccessor hiddenAccessor = HiddenFieldAccessorFactory.create(field); if (hiddenAccessor != null) { return hiddenAccessor; } - return createVarHandleAccessor(field); + return new InstanceAccessor(field); } - private static FieldAccessor createVarHandleAccessor(Field field) { - if (field.getType() == boolean.class) { - return new BooleanAccessor(field); - } else if (field.getType() == byte.class) { - return new ByteAccessor(field); - } else if (field.getType() == char.class) { - return new CharAccessor(field); - } else if (field.getType() == short.class) { - return new ShortAccessor(field); - } else if (field.getType() == int.class) { - return new IntAccessor(field); - } else if (field.getType() == long.class) { - return new LongAccessor(field); - } else if (field.getType() == float.class) { - return new FloatAccessor(field); - } else if (field.getType() == double.class) { - return new DoubleAccessor(field); - } else { - return new ObjectAccessor(field); - } + private static int accessKind(Field field) { + Class fieldType = field.getType(); + if (fieldType == boolean.class) { + return BOOLEAN_ACCESS; + } else if (fieldType == byte.class) { + return BYTE_ACCESS; + } else if (fieldType == char.class) { + return CHAR_ACCESS; + } else if (fieldType == short.class) { + return SHORT_ACCESS; + } else if (fieldType == int.class) { + return INT_ACCESS; + } else if (fieldType == long.class) { + return LONG_ACCESS; + } else if (fieldType == float.class) { + return FLOAT_ACCESS; + } else if (fieldType == double.class) { + return DOUBLE_ACCESS; + } + return OBJECT_ACCESS; } private static VarHandle fieldHandle(Field field) { @@ -85,462 +90,199 @@ private static IllegalStateException accessFailure(Field field, Throwable cause) cause); } - private static RuntimeException accessorFailure(Field field, Throwable cause) { - return new RuntimeException("Failed to access field: " + field, cause); - } - - private abstract static class InstanceAccessor extends FieldAccessor { - protected final VarHandle handle; + static final class InstanceAccessor extends FieldAccessor { + private final VarHandle handle; + private final int accessKind; InstanceAccessor(Field field) { super(field); handle = fieldHandle(field); - } - } - - /** Primitive boolean accessor. */ - public static class BooleanAccessor extends InstanceAccessor { - public BooleanAccessor(Field field) { - super(field); - Preconditions.checkArgument(field.getType() == boolean.class); + accessKind = accessKind(field); } @Override public Object get(Object obj) { - return getBoolean(obj); - } - - @Override - public boolean getBoolean(Object obj) { - return (boolean) handle.get(obj); - } - - @Override - public void set(Object obj, Object value) { - putBoolean(obj, (Boolean) value); - } - - @Override - public void putBoolean(Object obj, boolean value) { - try { - handle.set(obj, value); - } catch (RuntimeException e) { - throw accessorFailure(field, e); + switch (accessKind) { + case BOOLEAN_ACCESS: + return (boolean) handle.get(obj); + case BYTE_ACCESS: + return (byte) handle.get(obj); + case CHAR_ACCESS: + return (char) handle.get(obj); + case SHORT_ACCESS: + return (short) handle.get(obj); + case INT_ACCESS: + return (int) handle.get(obj); + case LONG_ACCESS: + return (long) handle.get(obj); + case FLOAT_ACCESS: + return (float) handle.get(obj); + case DOUBLE_ACCESS: + return (double) handle.get(obj); + case OBJECT_ACCESS: + return handle.get(obj); + default: + throw new IllegalStateException("Unsupported access kind " + accessKind); } } - } - - /** Primitive byte accessor. */ - public static class ByteAccessor extends InstanceAccessor { - public ByteAccessor(Field field) { - super(field); - Preconditions.checkArgument(field.getType() == byte.class); - } - - @Override - public Byte get(Object obj) { - return getByte(obj); - } - - @Override - public byte getByte(Object obj) { - return (byte) handle.get(obj); - } @Override public void set(Object obj, Object value) { - putByte(obj, (Byte) value); - } - - @Override - public void putByte(Object obj, byte value) { - try { - handle.set(obj, value); - } catch (RuntimeException e) { - throw accessorFailure(field, e); + switch (accessKind) { + case BOOLEAN_ACCESS: + handle.set(obj, (Boolean) value); + return; + case BYTE_ACCESS: + handle.set(obj, (Byte) value); + return; + case CHAR_ACCESS: + handle.set(obj, (Character) value); + return; + case SHORT_ACCESS: + handle.set(obj, (Short) value); + return; + case INT_ACCESS: + handle.set(obj, (Integer) value); + return; + case LONG_ACCESS: + handle.set(obj, (Long) value); + return; + case FLOAT_ACCESS: + handle.set(obj, (Float) value); + return; + case DOUBLE_ACCESS: + handle.set(obj, (Double) value); + return; + case OBJECT_ACCESS: + handle.set(obj, value); + return; + default: + throw new IllegalStateException("Unsupported access kind " + accessKind); } } - } - - /** Primitive char accessor. */ - public static class CharAccessor extends InstanceAccessor { - public CharAccessor(Field field) { - super(field); - Preconditions.checkArgument(field.getType() == char.class); - } @Override - public Character get(Object obj) { - return getChar(obj); - } - - @Override - public char getChar(Object obj) { - return (char) handle.get(obj); - } - - @Override - public void set(Object obj, Object value) { - putChar(obj, (Character) value); - } - - @Override - public void putChar(Object obj, char value) { - try { - handle.set(obj, value); - } catch (RuntimeException e) { - throw accessorFailure(field, e); - } - } - } - - /** Primitive short accessor. */ - public static class ShortAccessor extends InstanceAccessor { - public ShortAccessor(Field field) { - super(field); - Preconditions.checkArgument(field.getType() == short.class); - } - - @Override - public Short get(Object obj) { - return getShort(obj); - } - - @Override - public short getShort(Object obj) { - return (short) handle.get(obj); - } - - @Override - public void set(Object obj, Object value) { - putShort(obj, (Short) value); - } - - @Override - public void putShort(Object obj, short value) { - try { - handle.set(obj, value); - } catch (RuntimeException e) { - throw accessorFailure(field, e); + public void copy(Object sourceObject, Object targetObject) { + switch (accessKind) { + case BOOLEAN_ACCESS: + handle.set(targetObject, (boolean) handle.get(sourceObject)); + return; + case BYTE_ACCESS: + handle.set(targetObject, (byte) handle.get(sourceObject)); + return; + case CHAR_ACCESS: + handle.set(targetObject, (char) handle.get(sourceObject)); + return; + case SHORT_ACCESS: + handle.set(targetObject, (short) handle.get(sourceObject)); + return; + case INT_ACCESS: + handle.set(targetObject, (int) handle.get(sourceObject)); + return; + case LONG_ACCESS: + handle.set(targetObject, (long) handle.get(sourceObject)); + return; + case FLOAT_ACCESS: + handle.set(targetObject, (float) handle.get(sourceObject)); + return; + case DOUBLE_ACCESS: + handle.set(targetObject, (double) handle.get(sourceObject)); + return; + case OBJECT_ACCESS: + handle.set(targetObject, handle.get(sourceObject)); + return; + default: + throw new IllegalStateException("Unsupported access kind " + accessKind); } } - } - - /** Primitive int accessor. */ - public static class IntAccessor extends InstanceAccessor { - public IntAccessor(Field field) { - super(field); - Preconditions.checkArgument(field.getType() == int.class); - } - - @Override - public Integer get(Object obj) { - return getInt(obj); - } - - @Override - public int getInt(Object obj) { - return (int) handle.get(obj); - } - - @Override - public void set(Object obj, Object value) { - putInt(obj, (Integer) value); - } @Override - public void putInt(Object obj, int value) { - try { - handle.set(obj, value); - } catch (RuntimeException e) { - throw accessorFailure(field, e); - } - } - } - - /** Primitive long accessor. */ - public static class LongAccessor extends InstanceAccessor { - public LongAccessor(Field field) { - super(field); - Preconditions.checkArgument(field.getType() == long.class); - } - - @Override - public Long get(Object obj) { - return getLong(obj); - } - - @Override - public long getLong(Object obj) { - return (long) handle.get(obj); - } - - @Override - public void set(Object obj, Object value) { - putLong(obj, (Long) value); - } - - @Override - public void putLong(Object obj, long value) { - try { - handle.set(obj, value); - } catch (RuntimeException e) { - throw accessorFailure(field, e); - } - } - } - - /** Primitive float accessor. */ - public static class FloatAccessor extends InstanceAccessor { - public FloatAccessor(Field field) { - super(field); - Preconditions.checkArgument(field.getType() == float.class); - } - - @Override - public Object get(Object obj) { - return getFloat(obj); - } - - @Override - public float getFloat(Object obj) { - return (float) handle.get(obj); - } - - @Override - public void set(Object obj, Object value) { - putFloat(obj, (Float) value); - } - - @Override - public void putFloat(Object obj, float value) { - try { - handle.set(obj, value); - } catch (RuntimeException e) { - throw accessorFailure(field, e); - } - } - } - - /** Primitive double accessor. */ - public static class DoubleAccessor extends InstanceAccessor { - public DoubleAccessor(Field field) { - super(field); - Preconditions.checkArgument(field.getType() == double.class); - } - - @Override - public Object get(Object obj) { - return getDouble(obj); - } - - @Override - public double getDouble(Object obj) { - return (double) handle.get(obj); - } - - @Override - public void set(Object obj, Object value) { - putDouble(obj, (Double) value); - } - - @Override - public void putDouble(Object obj, double value) { - try { - handle.set(obj, value); - } catch (RuntimeException e) { - throw accessorFailure(field, e); - } - } - } - - /** Object accessor. */ - public static class ObjectAccessor extends InstanceAccessor { - public ObjectAccessor(Field field) { - super(field); - Preconditions.checkArgument(!TypeUtils.isPrimitive(field.getType())); - } - - @Override - public Object get(Object obj) { - return handle.get(obj); - } - - @Override - public void set(Object obj, Object value) { - try { - handle.set(obj, value); - } catch (RuntimeException e) { - throw accessorFailure(field, e); - } - } - } - - static final class GeneratedAccessor extends InstanceAccessor { - GeneratedAccessor(Field field) { - super(field); - } - - @Override - public Object get(Object obj) { - try { - return handle.get(obj); - } catch (RuntimeException e) { - throw accessorFailure(field, e); - } - } - - @Override - public void set(Object obj, Object value) { - try { - handle.set(obj, value); - } catch (RuntimeException e) { - throw accessorFailure(field, e); + public void copyObject(Object sourceObject, Object targetObject) { + if (accessKind == OBJECT_ACCESS) { + handle.set(targetObject, handle.get(sourceObject)); + } else { + super.copyObject(sourceObject, targetObject); } } @Override public boolean getBoolean(Object obj) { - try { - return (boolean) handle.get(obj); - } catch (RuntimeException e) { - throw accessorFailure(field, e); - } + return (boolean) handle.get(obj); } @Override public void putBoolean(Object obj, boolean value) { - try { - handle.set(obj, value); - } catch (RuntimeException e) { - throw accessorFailure(field, e); - } + handle.set(obj, value); } @Override public byte getByte(Object obj) { - try { - return (byte) handle.get(obj); - } catch (RuntimeException e) { - throw accessorFailure(field, e); - } + return (byte) handle.get(obj); } @Override public void putByte(Object obj, byte value) { - try { - handle.set(obj, value); - } catch (RuntimeException e) { - throw accessorFailure(field, e); - } + handle.set(obj, value); } @Override public char getChar(Object obj) { - try { - return (char) handle.get(obj); - } catch (RuntimeException e) { - throw accessorFailure(field, e); - } + return (char) handle.get(obj); } @Override public void putChar(Object obj, char value) { - try { - handle.set(obj, value); - } catch (RuntimeException e) { - throw accessorFailure(field, e); - } + handle.set(obj, value); } @Override public short getShort(Object obj) { - try { - return (short) handle.get(obj); - } catch (RuntimeException e) { - throw accessorFailure(field, e); - } + return (short) handle.get(obj); } @Override public void putShort(Object obj, short value) { - try { - handle.set(obj, value); - } catch (RuntimeException e) { - throw accessorFailure(field, e); - } + handle.set(obj, value); } @Override public int getInt(Object obj) { - try { - return (int) handle.get(obj); - } catch (RuntimeException e) { - throw accessorFailure(field, e); - } + return (int) handle.get(obj); } @Override public void putInt(Object obj, int value) { - try { - handle.set(obj, value); - } catch (RuntimeException e) { - throw accessorFailure(field, e); - } + handle.set(obj, value); } @Override public long getLong(Object obj) { - try { - return (long) handle.get(obj); - } catch (RuntimeException e) { - throw accessorFailure(field, e); - } + return (long) handle.get(obj); } @Override public void putLong(Object obj, long value) { - try { - handle.set(obj, value); - } catch (RuntimeException e) { - throw accessorFailure(field, e); - } + handle.set(obj, value); } @Override public float getFloat(Object obj) { - try { - return (float) handle.get(obj); - } catch (RuntimeException e) { - throw accessorFailure(field, e); - } + return (float) handle.get(obj); } @Override public void putFloat(Object obj, float value) { - try { - handle.set(obj, value); - } catch (RuntimeException e) { - throw accessorFailure(field, e); - } + handle.set(obj, value); } @Override public double getDouble(Object obj) { - try { - return (double) handle.get(obj); - } catch (RuntimeException e) { - throw accessorFailure(field, e); - } + return (double) handle.get(obj); } @Override public void putDouble(Object obj, double value) { - try { - handle.set(obj, value); - } catch (RuntimeException e) { - throw accessorFailure(field, e); - } + handle.set(obj, value); } } } From 3232a2aa693f79819ada4d3330088c37629e9dfa Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Tue, 2 Jun 2026 17:18:50 +0800 Subject: [PATCH 51/69] refactor(java): remove jdk25 hidden field accessors --- .agents/languages/java.md | 7 +- java/fory-core/pom.xml | 8 - .../fory/reflect/FieldAccessorStrategy.java | 4 - .../reflect/HiddenFieldAccessorFactory.java | 559 ------------------ .../builder/Jdk25MultiReleaseJarVerifier.java | 1 - .../fory/reflect/FieldAccessorTest.java | 9 - 6 files changed, 4 insertions(+), 584 deletions(-) delete mode 100644 java/fory-core/src/main/java25/org/apache/fory/reflect/HiddenFieldAccessorFactory.java diff --git a/.agents/languages/java.md b/.agents/languages/java.md index 44614889ff..7a6f2a190e 100644 --- a/.agents/languages/java.md +++ b/.agents/languages/java.md @@ -111,9 +111,10 @@ Load this file when changing anything under `java/` or when Java drives a cross- reflective `Field.set*` fallback. - JDK25+ `FieldAccessorStrategy` should use one final trusted-lookup `VarHandle` instance accessor with dense access-kind switches, not public primitive/object accessor classes or a JDK25 - `GeneratedAccessor`. Do not wrap `VarHandle.get/set` in hot-path try/catch blocks and do not call - `FieldAccessor.checkObj`; VarHandle validates null and receiver type itself. Root Unsafe offset - access still needs explicit receiver validation because Unsafe does not. + `GeneratedAccessor` or hidden-class accessor generation. Do not wrap `VarHandle.get/set` in + hot-path try/catch blocks and do not call `FieldAccessor.checkObj`; VarHandle validates null and + receiver type itself. Root Unsafe offset access still needs explicit receiver validation because + Unsafe does not. - Keep JDK26 `--illegal-final-field-mutation=deny` scoped to the JPMS runtime tests that prove Fory's field-access path. Do not put it in global Maven `JDK_JAVA_OPTIONS`, because build tools such as Lombok may perform their own reflective final-field access during compilation. diff --git a/java/fory-core/pom.xml b/java/fory-core/pom.xml index 7eb856d903..da608c772b 100644 --- a/java/fory-core/pom.xml +++ b/java/fory-core/pom.xml @@ -469,8 +469,6 @@ name="META-INF/versions/25/org/apache/fory/platform/internal/_UnsafeUtils.java"/> - @@ -488,9 +486,6 @@ - @@ -509,9 +504,6 @@ - diff --git a/java/fory-core/src/main/java25/org/apache/fory/reflect/FieldAccessorStrategy.java b/java/fory-core/src/main/java25/org/apache/fory/reflect/FieldAccessorStrategy.java index 99695c07da..97376a2471 100644 --- a/java/fory-core/src/main/java25/org/apache/fory/reflect/FieldAccessorStrategy.java +++ b/java/fory-core/src/main/java25/org/apache/fory/reflect/FieldAccessorStrategy.java @@ -40,10 +40,6 @@ private FieldAccessorStrategy() {} static FieldAccessor createAccessor(Field field) { Preconditions.checkArgument(!Modifier.isStatic(field.getModifiers()), field); - FieldAccessor hiddenAccessor = HiddenFieldAccessorFactory.create(field); - if (hiddenAccessor != null) { - return hiddenAccessor; - } return new InstanceAccessor(field); } diff --git a/java/fory-core/src/main/java25/org/apache/fory/reflect/HiddenFieldAccessorFactory.java b/java/fory-core/src/main/java25/org/apache/fory/reflect/HiddenFieldAccessorFactory.java deleted file mode 100644 index b9d88bbb5d..0000000000 --- a/java/fory-core/src/main/java25/org/apache/fory/reflect/HiddenFieldAccessorFactory.java +++ /dev/null @@ -1,559 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.fory.reflect; - -import java.io.ByteArrayOutputStream; -import java.io.DataOutputStream; -import java.io.IOException; -import java.lang.reflect.Constructor; -import java.lang.reflect.Field; -import java.lang.reflect.Modifier; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.atomic.AtomicInteger; -import org.apache.fory.platform.internal.DefineClass; - -final class HiddenFieldAccessorFactory { - private static final int JAVA5_CLASS_VERSION = 49; - private static final int ACC_PUBLIC = 0x0001; - private static final int ACC_FINAL = 0x0010; - private static final int ACC_SUPER = 0x0020; - private static final String FIELD_ACCESSOR = "org/apache/fory/reflect/FieldAccessor"; - private static final String FIELD_CTR_DESC = "(Ljava/lang/reflect/Field;)V"; - private static final AtomicInteger IDS = new AtomicInteger(); - - private HiddenFieldAccessorFactory() {} - - static FieldAccessor create(Field field) { - if (Modifier.isStatic(field.getModifiers()) || Modifier.isFinal(field.getModifiers())) { - return null; - } - try { - byte[] bytes = new AccessorClass(field).bytes(); - Class accessorClass = DefineClass.defineHiddenNestmate(field.getDeclaringClass(), bytes); - Constructor constructor = accessorClass.getDeclaredConstructor(Field.class); - constructor.setAccessible(true); - return (FieldAccessor) constructor.newInstance(field); - } catch (ReflectiveOperationException | RuntimeException | LinkageError e) { - return null; - } - } - - private static final class AccessorClass { - private final Field field; - private final Class fieldType; - private final String owner; - private final String thisClass; - private final String fieldDesc; - private final Pool pool = new Pool(); - - private int codeUtf8; - private int initName; - private int initDesc; - private int getName; - private int getDesc; - private int setName; - private int setDesc; - private int fieldRef; - private int ownerClass; - private int thisClassIndex; - private int fieldAccessorClass; - private int superCtr; - - private AccessorClass(Field field) { - this.field = field; - this.fieldType = field.getType(); - owner = internalName(field.getDeclaringClass()); - thisClass = owner + "$ForyFieldAccessor$" + IDS.incrementAndGet(); - fieldDesc = descriptor(fieldType); - } - - private byte[] bytes() { - try { - preparePool(); - List methods = methods(); - ByteArrayOutputStream bytes = new ByteArrayOutputStream(); - DataOutputStream out = new DataOutputStream(bytes); - out.writeInt(0xCAFEBABE); - out.writeShort(0); - out.writeShort(JAVA5_CLASS_VERSION); - pool.writeTo(out); - out.writeShort(ACC_FINAL | ACC_SUPER); - out.writeShort(thisClassIndex); - out.writeShort(fieldAccessorClass); - out.writeShort(0); - out.writeShort(0); - out.writeShort(methods.size()); - for (MethodDef method : methods) { - method.writeTo(out, codeUtf8); - } - out.writeShort(0); - return bytes.toByteArray(); - } catch (IOException e) { - throw new IllegalStateException("Failed to build field accessor bytecode", e); - } - } - - private void preparePool() { - codeUtf8 = pool.utf8("Code"); - initName = pool.utf8(""); - initDesc = pool.utf8(FIELD_CTR_DESC); - getName = pool.utf8("get"); - getDesc = pool.utf8("(Ljava/lang/Object;)Ljava/lang/Object;"); - setName = pool.utf8("set"); - setDesc = pool.utf8("(Ljava/lang/Object;Ljava/lang/Object;)V"); - thisClassIndex = pool.classInfo(thisClass); - ownerClass = pool.classInfo(owner); - fieldAccessorClass = pool.classInfo(FIELD_ACCESSOR); - superCtr = pool.methodRef(FIELD_ACCESSOR, "", FIELD_CTR_DESC); - fieldRef = pool.fieldRef(owner, field.getName(), fieldDesc); - if (fieldType.isPrimitive()) { - Primitive primitive = Primitive.of(fieldType); - pool.classInfo(primitive.wrapper); - pool.methodRef(primitive.wrapper, "valueOf", primitive.valueOfDesc); - pool.methodRef(primitive.wrapper, primitive.unboxName, primitive.unboxDesc); - } else { - pool.classInfo(castName(fieldType)); - } - } - - private List methods() throws IOException { - List methods = new ArrayList<>(); - methods.add(method(ACC_PUBLIC, initName, initDesc, 2, 2, constructorCode())); - methods.add(method(ACC_PUBLIC, getName, getDesc, 4, 2, objectGetterCode())); - if (fieldType.isPrimitive()) { - Primitive primitive = Primitive.of(fieldType); - methods.add( - method( - ACC_PUBLIC, - pool.utf8(primitive.getterName), - pool.utf8("(Ljava/lang/Object;)" + primitive.desc), - 3, - 2, - primitiveGetterCode(primitive))); - if (!Modifier.isFinal(field.getModifiers())) { - methods.add(method(ACC_PUBLIC, setName, setDesc, 4, 3, objectSetterCode(primitive))); - methods.add( - method( - ACC_PUBLIC, - pool.utf8(primitive.setterName), - pool.utf8("(Ljava/lang/Object;" + primitive.desc + ")V"), - 4, - primitive.maxLocals, - primitiveSetterCode(primitive))); - } - } else { - if (!Modifier.isFinal(field.getModifiers())) { - methods.add(method(ACC_PUBLIC, setName, setDesc, 3, 3, referenceSetterCode())); - } - } - return methods; - } - - private byte[] constructorCode() throws IOException { - Code code = new Code(); - code.u1(0x2A); // aload_0 - code.u1(0x2B); // aload_1 - code.u1(0xB7).u2(superCtr); // invokespecial - code.u1(0xB1); // return - return code.bytes(); - } - - private byte[] objectGetterCode() throws IOException { - Code code = directReadCode(); - if (fieldType.isPrimitive()) { - Primitive primitive = Primitive.of(fieldType); - code.u1(0xB8).u2(pool.methodRef(primitive.wrapper, "valueOf", primitive.valueOfDesc)); - } - code.u1(0xB0); // areturn - return code.bytes(); - } - - private byte[] primitiveGetterCode(Primitive primitive) throws IOException { - Code code = directReadCode(); - code.u1(primitive.returnOpcode); - return code.bytes(); - } - - private Code directReadCode() throws IOException { - Code code = new Code(); - code.u1(0x2B); // aload_1 - code.u1(0xC0).u2(ownerClass); // checkcast - code.u1(0xB4).u2(fieldRef); // getfield - return code; - } - - private byte[] objectSetterCode(Primitive primitive) throws IOException { - Code code = setterPrefix(); - code.u1(0x2C); // aload_2 - code.u1(0xC0).u2(pool.classInfo(primitive.wrapper)); - code.u1(0xB6).u2(pool.methodRef(primitive.wrapper, primitive.unboxName, primitive.unboxDesc)); - code.u1(0xB5).u2(fieldRef); // putfield - code.u1(0xB1); // return - return code.bytes(); - } - - private byte[] primitiveSetterCode(Primitive primitive) throws IOException { - Code code = setterPrefix(); - code.u1(primitive.loadOpcode); - code.u1(0xB5).u2(fieldRef); // putfield - code.u1(0xB1); // return - return code.bytes(); - } - - private byte[] referenceSetterCode() throws IOException { - Code code = setterPrefix(); - code.u1(0x2C); // aload_2 - code.u1(0xC0).u2(pool.classInfo(castName(fieldType))); - code.u1(0xB5).u2(fieldRef); // putfield - code.u1(0xB1); // return - return code.bytes(); - } - - private Code setterPrefix() throws IOException { - Code code = new Code(); - code.u1(0x2B); // aload_1 - code.u1(0xC0).u2(ownerClass); // checkcast - return code; - } - - private MethodDef method( - int access, - int name, - int desc, - int maxStack, - int maxLocals, - byte[] code) - throws IOException { - return new MethodDef(access, name, desc, maxStack, maxLocals, code); - } - } - - private static final class MethodDef { - private final int access; - private final int name; - private final int desc; - private final int maxStack; - private final int maxLocals; - private final byte[] code; - - private MethodDef(int access, int name, int desc, int maxStack, int maxLocals, byte[] code) { - this.access = access; - this.name = name; - this.desc = desc; - this.maxStack = maxStack; - this.maxLocals = maxLocals; - this.code = code; - } - - private void writeTo(DataOutputStream out, int codeUtf8) throws IOException { - out.writeShort(access); - out.writeShort(name); - out.writeShort(desc); - out.writeShort(1); - out.writeShort(codeUtf8); - out.writeInt(12 + code.length); - out.writeShort(maxStack); - out.writeShort(maxLocals); - out.writeInt(code.length); - out.write(code); - out.writeShort(0); - out.writeShort(0); - } - } - - private static String descriptor(Class type) { - if (type == void.class) { - return "V"; - } else if (type == boolean.class) { - return "Z"; - } else if (type == byte.class) { - return "B"; - } else if (type == char.class) { - return "C"; - } else if (type == short.class) { - return "S"; - } else if (type == int.class) { - return "I"; - } else if (type == long.class) { - return "J"; - } else if (type == float.class) { - return "F"; - } else if (type == double.class) { - return "D"; - } else if (type.isArray()) { - return type.getName().replace('.', '/'); - } - return "L" + internalName(type) + ";"; - } - - private static String castName(Class type) { - if (type.isArray()) { - return descriptor(type); - } - return internalName(type); - } - - private static String internalName(Class type) { - return type.getName().replace('.', '/'); - } - - private static final class Code { - private final ByteArrayOutputStream bytes = new ByteArrayOutputStream(); - private final DataOutputStream out = new DataOutputStream(bytes); - - private Code u1(int value) throws IOException { - out.writeByte(value); - return this; - } - - private Code u2(int value) throws IOException { - out.writeShort(value); - return this; - } - - private byte[] bytes() throws IOException { - out.flush(); - return bytes.toByteArray(); - } - } - - private static final class Pool { - private final ByteArrayOutputStream bytes = new ByteArrayOutputStream(); - private final DataOutputStream out = new DataOutputStream(bytes); - private int count = 1; - - private int utf8(String value) { - try { - int index = count++; - out.writeByte(1); - out.writeUTF(value); - return index; - } catch (IOException e) { - throw new IllegalStateException(e); - } - } - - private int classInfo(String name) { - try { - int nameIndex = utf8(name); - int index = count++; - out.writeByte(7); - out.writeShort(nameIndex); - return index; - } catch (IOException e) { - throw new IllegalStateException(e); - } - } - - private int nameAndType(String name, String desc) { - try { - int nameIndex = utf8(name); - int descIndex = utf8(desc); - int index = count++; - out.writeByte(12); - out.writeShort(nameIndex); - out.writeShort(descIndex); - return index; - } catch (IOException e) { - throw new IllegalStateException(e); - } - } - - private int fieldRef(String owner, String name, String desc) { - try { - int ownerIndex = classInfo(owner); - int nameAndTypeIndex = nameAndType(name, desc); - int index = count++; - out.writeByte(9); - out.writeShort(ownerIndex); - out.writeShort(nameAndTypeIndex); - return index; - } catch (IOException e) { - throw new IllegalStateException(e); - } - } - - private int methodRef(String owner, String name, String desc) { - try { - int ownerIndex = classInfo(owner); - int nameAndTypeIndex = nameAndType(name, desc); - int index = count++; - out.writeByte(10); - out.writeShort(ownerIndex); - out.writeShort(nameAndTypeIndex); - return index; - } catch (IOException e) { - throw new IllegalStateException(e); - } - } - - private void writeTo(DataOutputStream target) throws IOException { - out.flush(); - target.writeShort(count); - bytes.writeTo(target); - } - } - - private enum Primitive { - BOOLEAN( - boolean.class, - "Z", - "java/lang/Boolean", - "(Z)Ljava/lang/Boolean;", - "booleanValue", - "()Z", - "getBoolean", - "putBoolean", - 0x1C, - 0xAC, - 3), - BYTE( - byte.class, - "B", - "java/lang/Byte", - "(B)Ljava/lang/Byte;", - "byteValue", - "()B", - "getByte", - "putByte", - 0x1C, - 0xAC, - 3), - CHAR( - char.class, - "C", - "java/lang/Character", - "(C)Ljava/lang/Character;", - "charValue", - "()C", - "getChar", - "putChar", - 0x1C, - 0xAC, - 3), - SHORT( - short.class, - "S", - "java/lang/Short", - "(S)Ljava/lang/Short;", - "shortValue", - "()S", - "getShort", - "putShort", - 0x1C, - 0xAC, - 3), - INT( - int.class, - "I", - "java/lang/Integer", - "(I)Ljava/lang/Integer;", - "intValue", - "()I", - "getInt", - "putInt", - 0x1C, - 0xAC, - 3), - LONG( - long.class, - "J", - "java/lang/Long", - "(J)Ljava/lang/Long;", - "longValue", - "()J", - "getLong", - "putLong", - 0x20, - 0xAD, - 4), - FLOAT( - float.class, - "F", - "java/lang/Float", - "(F)Ljava/lang/Float;", - "floatValue", - "()F", - "getFloat", - "putFloat", - 0x24, - 0xAE, - 3), - DOUBLE( - double.class, - "D", - "java/lang/Double", - "(D)Ljava/lang/Double;", - "doubleValue", - "()D", - "getDouble", - "putDouble", - 0x28, - 0xAF, - 4); - - private final Class type; - private final String desc; - private final String wrapper; - private final String valueOfDesc; - private final String unboxName; - private final String unboxDesc; - private final String getterName; - private final String setterName; - private final int loadOpcode; - private final int returnOpcode; - private final int maxLocals; - - Primitive( - Class type, - String desc, - String wrapper, - String valueOfDesc, - String unboxName, - String unboxDesc, - String getterName, - String setterName, - int loadOpcode, - int returnOpcode, - int maxLocals) { - this.type = type; - this.desc = desc; - this.wrapper = wrapper; - this.valueOfDesc = valueOfDesc; - this.unboxName = unboxName; - this.unboxDesc = unboxDesc; - this.getterName = getterName; - this.setterName = setterName; - this.loadOpcode = loadOpcode; - this.returnOpcode = returnOpcode; - this.maxLocals = maxLocals; - } - - private static Primitive of(Class type) { - for (Primitive primitive : values()) { - if (primitive.type == type) { - return primitive; - } - } - throw new IllegalArgumentException("Not a primitive field type: " + type); - } - } -} diff --git a/java/fory-core/src/test/java/org/apache/fory/builder/Jdk25MultiReleaseJarVerifier.java b/java/fory-core/src/test/java/org/apache/fory/builder/Jdk25MultiReleaseJarVerifier.java index fc59c7c806..e27763be3d 100644 --- a/java/fory-core/src/test/java/org/apache/fory/builder/Jdk25MultiReleaseJarVerifier.java +++ b/java/fory-core/src/test/java/org/apache/fory/builder/Jdk25MultiReleaseJarVerifier.java @@ -52,7 +52,6 @@ public final class Jdk25MultiReleaseJarVerifier { "org/apache/fory/platform/internal/_UnsafeUtils.class", "org/apache/fory/builder/UnsafeCodegenSupport.class", "org/apache/fory/reflect/FieldAccessorStrategy.class", - "org/apache/fory/reflect/HiddenFieldAccessorFactory.class", "org/apache/fory/reflect/ConstructorBypassAllocator.class", "org/apache/fory/serializer/PlatformStringUtils.class" }; diff --git a/java/fory-core/src/test/java/org/apache/fory/reflect/FieldAccessorTest.java b/java/fory-core/src/test/java/org/apache/fory/reflect/FieldAccessorTest.java index d20309726c..e3daac37d0 100644 --- a/java/fory-core/src/test/java/org/apache/fory/reflect/FieldAccessorTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/reflect/FieldAccessorTest.java @@ -27,7 +27,6 @@ import lombok.AllArgsConstructor; import org.apache.fory.TestUtils; import org.apache.fory.platform.AndroidSupport; -import org.apache.fory.platform.JdkVersion; import org.apache.fory.reflect.FieldAccessorStrategy.GeneratedAccessor; import org.testng.Assert; import org.testng.annotations.Test; @@ -84,10 +83,6 @@ public void testHiddenAccessor() throws Exception { FieldAccessor finalAccessor = FieldAccessor.createAccessor(HiddenFields.class.getDeclaredField("finalValue")); Assert.assertEquals(finalAccessor.getLong(fields), 3L); - if (JdkVersion.MAJOR_VERSION >= 25) { - Assert.assertTrue(isHidden(intAccessor.getClass())); - Assert.assertTrue(isHidden(objectAccessor.getClass())); - } } @Test @@ -104,10 +99,6 @@ public void testFinalFieldWrites() throws Exception { Assert.assertEquals(objectAccessor.getObject(fields), "b"); } - private static boolean isHidden(Class cls) throws Exception { - return (Boolean) Class.class.getMethod("isHidden").invoke(cls); - } - @Test public void testAndroidReflectionFieldAccessorPaths() throws Exception { Process process = From 9879a8bc5af5a54c4f847ee827c90a9e12c01cc9 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Tue, 2 Jun 2026 17:58:11 +0800 Subject: [PATCH 52/69] fix(java): tighten jdk25 trusted lookup ownership --- .agents/languages/java.md | 9 +++- benchmarks/java/pom.xml | 8 --- integration_tests/jpms_tests/pom.xml | 1 - .../JpmsFieldAccessorTest.java | 28 ---------- .../src/main/java/org/apache/fory/Fory.java | 2 +- .../fory/platform/internal/_JDKAccess.java | 22 +++----- .../apache/fory/reflect/FieldAccessor.java | 13 +++-- .../fory/reflect/FieldAccessorFactory.java | 18 +++---- .../fory/reflect/FieldAccessorStrategy.java | 4 ++ .../fory/reflect/ObjectCreatorRegistry.java | 52 ------------------- .../apache/fory/reflect/ObjectCreators.java | 38 +++++++------- .../fory/reflect/ReflectionFieldAccessor.java | 2 - .../apache/fory/resolver/SharedRegistry.java | 35 +++++++++++-- .../apache/fory/resolver/TypeResolver.java | 2 +- .../serializer/ObjectStreamSerializer.java | 3 +- .../fory/reflect/ObjectCreatorsTest.java | 3 +- 16 files changed, 89 insertions(+), 151 deletions(-) delete mode 100644 java/fory-core/src/main/java/org/apache/fory/reflect/ObjectCreatorRegistry.java diff --git a/.agents/languages/java.md b/.agents/languages/java.md index 7a6f2a190e..3ad46bc0be 100644 --- a/.agents/languages/java.md +++ b/.agents/languages/java.md @@ -60,6 +60,7 @@ Load this file when changing anything under `java/` or when Java drives a cross- - Put latest-JDK or virtual-thread tests in the latest-JDK test modules with the matching compiler/profile floor, and centralize runtime-version probing in existing compatibility utilities. - For JDK25+ zero-Unsafe work, preserve serializer-family selection by type and configuration. Do not switch a type from `ObjectStreamSerializer` or another Fory serializer family to `JavaSerializer`, a JDK stream fallback, or any broad `java.* Serializable` fallback by JDK version or no-arg-constructor shape. - JDK25+ zero-Unsafe runtime support is a JPMS named-module design. Fory core must run with `--add-opens=java.base/java.lang.invoke=org.apache.fory.core` so it can obtain the trusted lookup; missing this open is an invalid access configuration, not a reason to open per-package JDK internals or switch serializer/object-creation families. Do not use `ALL-UNNAMED` as the zero-Unsafe verification target. +- Do not probe JDK25+ trusted-lookup availability and turn `_JDKAccess` field-access booleans false when the required `java.base/java.lang.invoke` open is missing. Keep those access flags true on JDK25+ and let the owning trusted-lookup path raise the configuration error. - Keep JDK25+ unsafe-removal implementation invariants in agent/design docs and tests, not user guides. User guides should document user actions such as `--sun-misc-unsafe-memory-access=deny`, `java.base/java.lang.invoke` opens, package opens for user named modules, and public APIs such as `@ForyConstructor` and `BaseFory.registerConstructor(...)`; do not expose internal serializer names, owner-model rationale, or avoided fallback strategies there. - JDK25+ zero-Unsafe final-field writes must use a true target-class trusted lookup from the original `IMPL_LOOKUP`, not `IMPL_LOOKUP.in(type)`. JDK26+ normal Fory final-field restoration must pass with `--illegal-final-field-mutation=deny` and must not require `--enable-final-field-mutation`. - For JDK25+ object creation, do not use `sun.reflect.ReflectionFactory` or `jdk.internal.reflect.ReflectionFactory`. The shared `ObjectCreators` facade should route ObjectStream-compatible construction through `ParentNoArgCtrObjectCreator`; the replaceable constructor-bypass allocator owns the JDK8-24 Unsafe allocation path and the JDK25+ `ObjectStreamClass.newInstance` path. Serializable classes without a no-arg constructor may use that trusted-lookup owner; non-Serializable classes without a no-arg constructor require `@ForyConstructor`, `BaseFory.registerConstructor(...)`, record construction, or a custom serializer. @@ -76,6 +77,10 @@ Load this file when changing anything under `java/` or when Java drives a cross- non-Serializable superclass no-arg constructor, and private no-arg constructors must be rejected instead of made accessible. Package-private no-arg constructors are valid only for same-package Serializable subclasses. +- Runtime object creator caches are `SharedRegistry` state backed by `ConcurrentHashMap`. + Registered constructors should store the resulting `ObjectCreator` directly there, not through a + soft cache or a separate registry class. Keep ObjectStream-specific creators separate from normal + constructor-bound object creators. - Root codegen and builder classes that still need Unsafe on JDK8-24 must route symbolic Unsafe access through a helper with a Java 25 replacement. Do not leave `_JDKAccess.unsafe()` or `sun.misc.Unsafe` references in JDK25-visible classes outside matching `java25` replacements. @@ -113,8 +118,8 @@ Load this file when changing anything under `java/` or when Java drives a cross- with dense access-kind switches, not public primitive/object accessor classes or a JDK25 `GeneratedAccessor` or hidden-class accessor generation. Do not wrap `VarHandle.get/set` in hot-path try/catch blocks and do not call `FieldAccessor.checkObj`; VarHandle validates null and - receiver type itself. Root Unsafe offset access still needs explicit receiver validation because - Unsafe does not. + receiver type itself. Root Unsafe offset access may keep a debug-only `assert` receiver check + because Unsafe does not validate the target object; do not add production receiver checks. - Keep JDK26 `--illegal-final-field-mutation=deny` scoped to the JPMS runtime tests that prove Fory's field-access path. Do not put it in global Maven `JDK_JAVA_OPTIONS`, because build tools such as Lombok may perform their own reflective final-field access during compilation. diff --git a/benchmarks/java/pom.xml b/benchmarks/java/pom.xml index b04daf401e..812906d202 100644 --- a/benchmarks/java/pom.xml +++ b/benchmarks/java/pom.xml @@ -269,8 +269,6 @@ name="META-INF/versions/25/org/apache/fory/platform/internal/_UnsafeUtils.class"/> - @@ -293,9 +291,6 @@ - @@ -317,9 +312,6 @@ - diff --git a/integration_tests/jpms_tests/pom.xml b/integration_tests/jpms_tests/pom.xml index dbb03218e3..e796a80e9b 100644 --- a/integration_tests/jpms_tests/pom.xml +++ b/integration_tests/jpms_tests/pom.xml @@ -87,7 +87,6 @@ --sun-misc-unsafe-memory-access=deny --add-opens=java.base/java.lang.invoke=org.apache.fory.core - --add-opens=org.apache.fory.core/org.apache.fory.platform.internal=org.apache.fory.integration_tests ${fory.final.field.policy.arg} diff --git a/integration_tests/jpms_tests/src/test/java/org/apache/fory/integration_tests/JpmsFieldAccessorTest.java b/integration_tests/jpms_tests/src/test/java/org/apache/fory/integration_tests/JpmsFieldAccessorTest.java index b2cc8299b8..9ce67d4161 100644 --- a/integration_tests/jpms_tests/src/test/java/org/apache/fory/integration_tests/JpmsFieldAccessorTest.java +++ b/integration_tests/jpms_tests/src/test/java/org/apache/fory/integration_tests/JpmsFieldAccessorTest.java @@ -19,12 +19,7 @@ package org.apache.fory.integration_tests; -import java.lang.invoke.MethodHandle; -import java.lang.invoke.MethodHandles; -import java.lang.invoke.MethodType; -import java.lang.invoke.VarHandle; import java.lang.reflect.Field; -import java.lang.reflect.Method; import org.apache.fory.Fory; import org.apache.fory.integration_tests.constructor.PrivateConstructorBean; import org.apache.fory.integration_tests.model.PrivateFieldBean; @@ -56,20 +51,6 @@ public void testPrivateFinalFieldSerialization() { Assert.assertEquals(result.value(), 13); } - @Test - public void testTrustedLookupFinalWrite() throws Throwable { - PrivateFieldBean bean = new PrivateFieldBean(17); - MethodHandles.Lookup lookup = trustedLookup(PrivateFieldBean.class); - VarHandle handle = lookup.findVarHandle(PrivateFieldBean.class, "value", int.class); - handle.set(bean, 19); - Assert.assertEquals(bean.value(), 19); - - MethodHandle setter = - lookup.findSetter(PrivateFieldBean.class, "value", int.class).asType(setterType()); - setter.invokeExact(bean, 23); - Assert.assertEquals(bean.value(), 23); - } - @Test public void testReflectionFinalWriteDenied() throws Exception { if (JDK_MAJOR_VERSION < 26) { @@ -106,13 +87,4 @@ public void testPublicSerializerInExportedPackage() { Assert.assertEquals(result.value, 11); } - private static MethodHandles.Lookup trustedLookup(Class type) throws Exception { - Class jdkAccess = Class.forName("org.apache.fory.platform.internal._JDKAccess"); - Method method = jdkAccess.getMethod("_trustedLookup", Class.class); - return (MethodHandles.Lookup) method.invoke(null, type); - } - - private static MethodType setterType() { - return MethodType.methodType(void.class, PrivateFieldBean.class, int.class); - } } diff --git a/java/fory-core/src/main/java/org/apache/fory/Fory.java b/java/fory-core/src/main/java/org/apache/fory/Fory.java index 81b8eeabdb..77d32c0ba1 100644 --- a/java/fory-core/src/main/java/org/apache/fory/Fory.java +++ b/java/fory-core/src/main/java/org/apache/fory/Fory.java @@ -224,7 +224,7 @@ public void registerConstructor( + " after its serializer has been created. Register constructors before calling " + "`getSerializer`, `serialize`, `deserialize`, or `copy` for that type."); } - sharedRegistry.getObjectCreatorRegistry().registerConstructor(type, constructor, fieldNames); + sharedRegistry.registerConstructor(type, constructor, fieldNames); } @Override diff --git a/java/fory-core/src/main/java/org/apache/fory/platform/internal/_JDKAccess.java b/java/fory-core/src/main/java/org/apache/fory/platform/internal/_JDKAccess.java index 635b61f1d5..31792c06bf 100644 --- a/java/fory-core/src/main/java/org/apache/fory/platform/internal/_JDKAccess.java +++ b/java/fory-core/src/main/java/org/apache/fory/platform/internal/_JDKAccess.java @@ -70,12 +70,13 @@ public class _JDKAccess { JDK_CONCURRENT_FIELD_ACCESS = false; JDK_PROXY_FIELD_ACCESS = false; } else if (JdkVersion.MAJOR_VERSION >= 25) { - boolean trustedLookupAvailable = trustedLookupAvailable(); - JDK_INTERNAL_FIELD_ACCESS = trustedLookupAvailable; - JDK_LANG_FIELD_ACCESS = trustedLookupAvailable; - JDK_COLLECTION_FIELD_ACCESS = trustedLookupAvailable; - JDK_CONCURRENT_FIELD_ACCESS = trustedLookupAvailable; - JDK_PROXY_FIELD_ACCESS = trustedLookupAvailable; + // JDK25+ zero-Unsafe mode requires java.base/java.lang.invoke to be opened to fory-core. + // Missing that open is an invalid runtime configuration, not a fallback signal. + JDK_INTERNAL_FIELD_ACCESS = true; + JDK_LANG_FIELD_ACCESS = true; + JDK_COLLECTION_FIELD_ACCESS = true; + JDK_CONCURRENT_FIELD_ACCESS = true; + JDK_PROXY_FIELD_ACCESS = true; } else { JDK_INTERNAL_FIELD_ACCESS = true; JDK_LANG_FIELD_ACCESS = true; @@ -85,15 +86,6 @@ public class _JDKAccess { } } - private static boolean trustedLookupAvailable() { - try { - _Lookup._trustedLookup(Object.class); - return true; - } catch (Throwable ignored) { - return false; - } - } - private static final ClassValueCache lookupCache = ClassValueCache.newClassKeyCache(32); // CHECKSTYLE.OFF:MethodName diff --git a/java/fory-core/src/main/java/org/apache/fory/reflect/FieldAccessor.java b/java/fory-core/src/main/java/org/apache/fory/reflect/FieldAccessor.java index 88c12d4201..4fbbf54f88 100644 --- a/java/fory-core/src/main/java/org/apache/fory/reflect/FieldAccessor.java +++ b/java/fory-core/src/main/java/org/apache/fory/reflect/FieldAccessor.java @@ -23,7 +23,6 @@ import org.apache.fory.util.Preconditions; /** Field accessor for primitive types and object types. */ -@SuppressWarnings({"unchecked", "rawtypes"}) public abstract class FieldAccessor { private static final int BOOLEAN_ACCESS = 1; private static final int BYTE_ACCESS = 2; @@ -183,10 +182,14 @@ public Object getObject(Object targetObject) { return get(targetObject); } - void checkObj(Object obj) { - if (!this.field.getDeclaringClass().isAssignableFrom(obj.getClass())) { - throw new IllegalArgumentException("Illegal class " + obj.getClass()); - } + final void checkObj(Object obj) { + // Unsafe offset access does not validate the receiver. A wrong receiver is a Fory + // programming error, so keep this debug-only instead of adding production hot-path checks. + assert field.getDeclaringClass().isInstance(obj) : illegalObject(obj); + } + + private String illegalObject(Object obj) { + return "Illegal class " + (obj == null ? null : obj.getClass()); } @Override diff --git a/java/fory-core/src/main/java/org/apache/fory/reflect/FieldAccessorFactory.java b/java/fory-core/src/main/java/org/apache/fory/reflect/FieldAccessorFactory.java index 782063fef3..a9d5302482 100644 --- a/java/fory-core/src/main/java/org/apache/fory/reflect/FieldAccessorFactory.java +++ b/java/fory-core/src/main/java/org/apache/fory/reflect/FieldAccessorFactory.java @@ -120,7 +120,7 @@ public void set(Object obj, Object value) { } } - public static class BooleanGetter extends FieldGetter { + private static class BooleanGetter extends FieldGetter { private final Predicate getter; public BooleanGetter(Field field, Predicate getter) { @@ -141,7 +141,7 @@ public boolean getBoolean(Object obj) { } } - public static class ByteGetter extends FieldGetter { + private static class ByteGetter extends FieldGetter { private final ToByteFunction getter; public ByteGetter(Field field, ToByteFunction getter) { @@ -161,7 +161,7 @@ public byte getByte(Object obj) { } } - public static class CharGetter extends FieldGetter { + private static class CharGetter extends FieldGetter { private final ToCharFunction getter; public CharGetter(Field field, ToCharFunction getter) { @@ -181,7 +181,7 @@ public char getChar(Object obj) { } } - public static class ShortGetter extends FieldGetter { + private static class ShortGetter extends FieldGetter { private final ToShortFunction getter; public ShortGetter(Field field, ToShortFunction getter) { @@ -201,7 +201,7 @@ public short getShort(Object obj) { } } - public static class IntGetter extends FieldGetter { + private static class IntGetter extends FieldGetter { private final ToIntFunction getter; public IntGetter(Field field, ToIntFunction getter) { @@ -221,7 +221,7 @@ public int getInt(Object obj) { } } - public static class LongGetter extends FieldGetter { + private static class LongGetter extends FieldGetter { private final ToLongFunction getter; public LongGetter(Field field, ToLongFunction getter) { @@ -241,7 +241,7 @@ public long getLong(Object obj) { } } - public static class FloatGetter extends FieldGetter { + private static class FloatGetter extends FieldGetter { private final ToFloatFunction getter; public FloatGetter(Field field, ToFloatFunction getter) { @@ -261,7 +261,7 @@ public float getFloat(Object obj) { } } - public static class DoubleGetter extends FieldGetter { + private static class DoubleGetter extends FieldGetter { private final ToDoubleFunction getter; public DoubleGetter(Field field, ToDoubleFunction getter) { @@ -281,7 +281,7 @@ public double getDouble(Object obj) { } } - public static class ObjectGetter extends FieldGetter { + private static class ObjectGetter extends FieldGetter { private final Function getter; public ObjectGetter(Field field, Function getter) { diff --git a/java/fory-core/src/main/java/org/apache/fory/reflect/FieldAccessorStrategy.java b/java/fory-core/src/main/java/org/apache/fory/reflect/FieldAccessorStrategy.java index 2ab34a3625..9ac559fe6b 100644 --- a/java/fory-core/src/main/java/org/apache/fory/reflect/FieldAccessorStrategy.java +++ b/java/fory-core/src/main/java/org/apache/fory/reflect/FieldAccessorStrategy.java @@ -165,6 +165,8 @@ public void set(Object obj, Object value) { @Override public void copy(Object sourceObject, Object targetObject) { + checkObj(sourceObject); + checkObj(targetObject); switch (accessKind) { case BOOLEAN_ACCESS: UNSAFE.putBoolean( @@ -201,6 +203,8 @@ public void copy(Object sourceObject, Object targetObject) { @Override public void copyObject(Object sourceObject, Object targetObject) { + checkObj(sourceObject); + checkObj(targetObject); if (accessKind == OBJECT_ACCESS) { UNSAFE.putObject(targetObject, fieldOffset, UNSAFE.getObject(sourceObject, fieldOffset)); } else { diff --git a/java/fory-core/src/main/java/org/apache/fory/reflect/ObjectCreatorRegistry.java b/java/fory-core/src/main/java/org/apache/fory/reflect/ObjectCreatorRegistry.java deleted file mode 100644 index a1e520727a..0000000000 --- a/java/fory-core/src/main/java/org/apache/fory/reflect/ObjectCreatorRegistry.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.fory.reflect; - -import java.lang.reflect.Constructor; -import org.apache.fory.annotation.Internal; -import org.apache.fory.collection.ClassValueCache; - -/** Runtime-scoped object creator cache and explicit constructor registrations. */ -@Internal -public final class ObjectCreatorRegistry { - private final ClassValueCache> objectCreatorCache = - ClassValueCache.newClassKeySoftCache(8); - private final ClassValueCache> constructorMatches = - ClassValueCache.newClassKeyCache(8); - - public ObjectCreator getObjectCreator(Class type) { - return (ObjectCreator) - objectCreatorCache.get( - type, - () -> - ObjectCreators.createObjectCreator( - type, - (ObjectCreators.ConstructorMatch) constructorMatches.getIfPresent(type))); - } - - public void registerConstructor( - Class type, Constructor constructor, String... fieldNames) { - ObjectCreators.ConstructorMatch match = - ObjectCreators.explicitConstructor(type, constructor, fieldNames.clone(), "registered"); - ObjectCreator objectCreator = ObjectCreators.createObjectCreator(type, match); - constructorMatches.put(type, match); - objectCreatorCache.put(type, objectCreator); - } -} diff --git a/java/fory-core/src/main/java/org/apache/fory/reflect/ObjectCreators.java b/java/fory-core/src/main/java/org/apache/fory/reflect/ObjectCreators.java index 108f2c01cb..386ac77862 100644 --- a/java/fory-core/src/main/java/org/apache/fory/reflect/ObjectCreators.java +++ b/java/fory-core/src/main/java/org/apache/fory/reflect/ObjectCreators.java @@ -44,13 +44,12 @@ import org.apache.fory.platform.GraalvmSupport; import org.apache.fory.platform.JdkVersion; import org.apache.fory.platform.internal._JDKAccess; -import org.apache.fory.resolver.TypeResolver; import org.apache.fory.type.Descriptor; import org.apache.fory.type.TypeUtils; import org.apache.fory.util.record.RecordUtils; /** - * Factory class for creating and caching {@link ObjectCreator} instances. + * Factory class for creating {@link ObjectCreator} instances. * *

    This class provides a centralized way to obtain optimized object creators for different types. * It automatically selects the most appropriate creation strategy based on the target type and @@ -67,8 +66,10 @@ * constructors, and throws when no supported reflective construction path exists * * - *

    All created ObjectCreator instances are cached using a soft reference cache to improve - * performance on repeated requests for the same type. + *

    The static {@link #getObjectCreator(Class)} method keeps the legacy process-global cache. + * Runtime-owned paths should use {@link + * org.apache.fory.resolver.TypeResolver#getObjectCreator(Class)} so constructor registrations and + * ObjectStream-compatible creators stay scoped to the Fory runtime. * *

    Thread Safety: This class and all returned ObjectCreator instances are * thread-safe and can be safely used across multiple threads concurrently. @@ -77,8 +78,6 @@ public class ObjectCreators { private static final ClassValueCache> cache = ClassValueCache.newClassKeySoftCache(8); - private static final ClassValueCache> objectStreamCache = - ClassValueCache.newClassKeySoftCache(8); /** * Returns an optimized ObjectCreator for the given type. @@ -97,20 +96,10 @@ public static ObjectCreator getObjectCreator(Class type) { return (ObjectCreator) cache.get(type, () -> createObjectCreator(type, null)); } - public static ObjectCreator getObjectCreator(TypeResolver typeResolver, Class type) { - return typeResolver.getSharedRegistry().getObjectCreatorRegistry().getObjectCreator(type); - } - - /** - * Returns the creator used by Java ObjectStream-compatible serializers. - * - *

    ObjectStream serializers reconstruct an empty instance before applying stream fields. - * Serializable class constructors and constructor mappings from {@link ForyConstructor} or {@code - * BaseFory.registerConstructor} are not semantically valid for this path. - */ + /** Creates an uncached object creator for runtime-scoped registries. */ @Internal - public static ObjectCreator getObjectStreamCreator(Class type) { - return (ObjectCreator) objectStreamCache.get(type, () -> createObjectStreamCreator(type)); + public static ObjectCreator createObjectCreator(Class type) { + return createObjectCreator(type, null); } static ObjectCreator createObjectCreator( @@ -143,7 +132,16 @@ static ObjectCreator createObjectCreator( return new DeclaredNoArgCtrObjectCreator<>(type); } - private static ObjectCreator createObjectStreamCreator(Class type) { + /** Creates an uncached constructor-bound object creator for runtime-scoped registries. */ + @Internal + public static ObjectCreator createObjectCreator( + Class type, Constructor constructor, String[] fieldNames, String source) { + return createObjectCreator(type, explicitConstructor(type, constructor, fieldNames, source)); + } + + /** Creates an uncached empty-instance creator for Java ObjectStream-compatible serializers. */ + @Internal + public static ObjectCreator createObjectStreamCreator(Class type) { if (AndroidSupport.IS_ANDROID) { Constructor noArgConstructor = ReflectionUtils.getNoArgConstructor(type); if (noArgConstructor != null) { diff --git a/java/fory-core/src/main/java/org/apache/fory/reflect/ReflectionFieldAccessor.java b/java/fory-core/src/main/java/org/apache/fory/reflect/ReflectionFieldAccessor.java index 82ffe00e47..25e43a929e 100644 --- a/java/fory-core/src/main/java/org/apache/fory/reflect/ReflectionFieldAccessor.java +++ b/java/fory-core/src/main/java/org/apache/fory/reflect/ReflectionFieldAccessor.java @@ -34,7 +34,6 @@ final class ReflectionFieldAccessor extends FieldAccessor { @Override public Object get(Object obj) { - checkObj(obj); try { return field.get(obj); } catch (IllegalAccessException | IllegalArgumentException e) { @@ -44,7 +43,6 @@ public Object get(Object obj) { @Override public void set(Object obj, Object value) { - checkObj(obj); try { field.set(obj, value); } catch (IllegalAccessException | IllegalArgumentException e) { diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/SharedRegistry.java b/java/fory-core/src/main/java/org/apache/fory/resolver/SharedRegistry.java index 58863c88d9..a2a8ecebd5 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/SharedRegistry.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/SharedRegistry.java @@ -19,6 +19,7 @@ package org.apache.fory.resolver; +import java.lang.reflect.Constructor; import java.lang.reflect.Member; import java.util.ArrayList; import java.util.Collections; @@ -39,7 +40,8 @@ import org.apache.fory.meta.MetaStringEncoder; import org.apache.fory.meta.TypeDef; import org.apache.fory.platform.GraalvmSupport; -import org.apache.fory.reflect.ObjectCreatorRegistry; +import org.apache.fory.reflect.ObjectCreator; +import org.apache.fory.reflect.ObjectCreators; import org.apache.fory.serializer.Serializer; import org.apache.fory.type.Descriptor; import org.apache.fory.type.DescriptorGrouper; @@ -82,7 +84,10 @@ public final class SharedRegistry { new ConcurrentIdentityMap<>(); final ConcurrentIdentityMap, Serializer> registeredSerializerCache = new ConcurrentIdentityMap<>(); - private final ObjectCreatorRegistry objectCreatorRegistry = new ObjectCreatorRegistry(); + private final ConcurrentHashMap, ObjectCreator> objectCreatorCache = + new ConcurrentHashMap<>(); + private final ConcurrentHashMap, ObjectCreator> objectStreamCreatorCache = + new ConcurrentHashMap<>(); final StaticGeneratedSerializerRegistry staticGeneratedSerializerRegistry = new StaticGeneratedSerializerRegistry(); private final Object metaStringCacheLock = new Object(); @@ -127,8 +132,30 @@ Serializer cacheRegisteredSerializer(Class type, Serializer serializer) return existing; } - public ObjectCreatorRegistry getObjectCreatorRegistry() { - return objectCreatorRegistry; + @SuppressWarnings("unchecked") + public ObjectCreator getObjectCreator(Class type) { + return (ObjectCreator) + objectCreatorCache.computeIfAbsent(type, ObjectCreators::createObjectCreator); + } + + /** + * Returns the runtime-scoped creator used by Java ObjectStream-compatible serializers. + * + *

    ObjectStream reconstruction creates an empty instance before stream fields are read, so + * explicit constructor mappings registered for normal object serializers are not semantically + * valid for this path. + */ + @SuppressWarnings("unchecked") + public ObjectCreator getObjectStreamCreator(Class type) { + return (ObjectCreator) + objectStreamCreatorCache.computeIfAbsent(type, ObjectCreators::createObjectStreamCreator); + } + + public void registerConstructor( + Class type, Constructor constructor, String... fieldNames) { + objectCreatorCache.put( + type, + ObjectCreators.createObjectCreator(type, constructor, fieldNames.clone(), "registered")); } TypeInfo cacheRegisteredTypeInfo(Class type, TypeInfo typeInfo) { diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java index 3288370cd2..4ff3817624 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java @@ -350,7 +350,7 @@ public abstract void registerEnum( */ @Internal public final ObjectCreator getObjectCreator(Class type) { - return sharedRegistry.getObjectCreatorRegistry().getObjectCreator(type); + return sharedRegistry.getObjectCreator(type); } /** diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectStreamSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectStreamSerializer.java index a72c1e78dc..2075a193c3 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectStreamSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectStreamSerializer.java @@ -65,7 +65,6 @@ import org.apache.fory.platform.AndroidSupport; import org.apache.fory.platform.GraalvmSupport; import org.apache.fory.platform.internal._JDKAccess; -import org.apache.fory.reflect.ObjectCreators; import org.apache.fory.resolver.ClassResolver; import org.apache.fory.resolver.TypeInfo; import org.apache.fory.resolver.TypeResolver; @@ -186,7 +185,7 @@ private static ObjectStreamClass safeObjectStreamClassLookup(Class type) { } public ObjectStreamSerializer(TypeResolver typeResolver, Class type) { - super(typeResolver, type, ObjectCreators.getObjectStreamCreator(type)); + super(typeResolver, type, typeResolver.getSharedRegistry().getObjectStreamCreator(type)); if (!Serializable.class.isAssignableFrom(type)) { throw new IllegalArgumentException( String.format("Class %s should implement %s.", type, Serializable.class)); diff --git a/java/fory-core/src/test/java/org/apache/fory/reflect/ObjectCreatorsTest.java b/java/fory-core/src/test/java/org/apache/fory/reflect/ObjectCreatorsTest.java index 7bdae7cf71..118bbad57b 100644 --- a/java/fory-core/src/test/java/org/apache/fory/reflect/ObjectCreatorsTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/reflect/ObjectCreatorsTest.java @@ -30,6 +30,7 @@ import org.apache.fory.platform.AndroidSupport; import org.apache.fory.platform.JdkVersion; import org.apache.fory.reflect.ObjectCreators.ParentNoArgCtrObjectCreator; +import org.apache.fory.resolver.SharedRegistry; import org.testng.Assert; import org.testng.annotations.Test; @@ -72,7 +73,7 @@ public void testFailedCtorRegNotPublished() throws Exception { if (JdkVersion.MAJOR_VERSION < 9) { return; } - ObjectCreatorRegistry registry = new ObjectCreatorRegistry(); + SharedRegistry registry = new SharedRegistry(); Constructor constructor = String.class.getDeclaredConstructor(byte[].class, byte.class); Assert.assertThrows( ForyException.class, From bb78ac7c66ef9f4fe9f54da8ef5154001a3ecf77 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Tue, 2 Jun 2026 18:19:42 +0800 Subject: [PATCH 53/69] refactor(java): rename field accessor owners --- .agents/languages/java.md | 13 ++++++++++-- benchmarks/java/pom.xml | 10 +++++----- .../fory/benchmark/Jdk25MrJarCheck.java | 2 +- java/fory-core/pom.xml | 10 +++++----- .../fory/builder/BaseObjectCodecBuilder.java | 6 ++---- .../apache/fory/reflect/FieldAccessor.java | 14 ++++++++++++- ...ategy.java => InstanceFieldAccessors.java} | 4 ++-- ...Factory.java => RecordFieldAccessors.java} | 20 ++++--------------- ...ategy.java => InstanceFieldAccessors.java} | 4 ++-- .../fory-core/native-image.properties | 4 ++-- .../builder/Jdk25MultiReleaseJarVerifier.java | 2 +- .../fory/reflect/FieldAccessorTest.java | 2 +- 12 files changed, 49 insertions(+), 42 deletions(-) rename java/fory-core/src/main/java/org/apache/fory/reflect/{FieldAccessorStrategy.java => InstanceFieldAccessors.java} (99%) rename java/fory-core/src/main/java/org/apache/fory/reflect/{FieldAccessorFactory.java => RecordFieldAccessors.java} (91%) rename java/fory-core/src/main/java25/org/apache/fory/reflect/{FieldAccessorStrategy.java => InstanceFieldAccessors.java} (99%) diff --git a/.agents/languages/java.md b/.agents/languages/java.md index 3ad46bc0be..f3b631a6a7 100644 --- a/.agents/languages/java.md +++ b/.agents/languages/java.md @@ -81,6 +81,12 @@ Load this file when changing anything under `java/` or when Java drives a cross- Registered constructors should store the resulting `ObjectCreator` directly there, not through a soft cache or a separate registry class. Keep ObjectStream-specific creators separate from normal constructor-bound object creators. +- Generated Fory object serializers must initialize object-creator fields through + `TypeResolver.getObjectCreator(Class)`, so generated code respects records, `@ForyConstructor`, + and `BaseFory.registerConstructor(...)`. Do not emit generated calls to + `ObjectCreators.getObjectCreator(TypeResolver, Class)` or bypass the runtime-scoped owner; format + builders without a Fory runtime context may use the base `ObjectCreators.getObjectCreator(Class)` + construction default. - Root codegen and builder classes that still need Unsafe on JDK8-24 must route symbolic Unsafe access through a helper with a Java 25 replacement. Do not leave `_JDKAccess.unsafe()` or `sun.misc.Unsafe` references in JDK25-visible classes outside matching `java25` replacements. @@ -107,14 +113,17 @@ Load this file when changing anything under `java/` or when Java drives a cross- - JDK25+ `PlatformStringUtils` getter methods sit behind `StringSerializer` static-final access gates. Do not add per-call access checks in those getters; missing module opens should fail at trusted-lookup initialization or cold setup, not inside string hot paths. -- JDK25+ `FieldAccessorStrategy` owns instance field access only. Do not add static-field handling +- `FieldAccessor` owns field-accessor dispatch. `RecordFieldAccessors` owns record field access, + and `InstanceFieldAccessors` owns non-record instance field access. Do not reintroduce a + `FieldAccessorFactory` layer. +- JDK25+ `InstanceFieldAccessors` owns instance field access only. Do not add static-field handling or per-write reflection fallback there, and do not expose public `FieldAccessor` static-field factories. Static special cases such as Scala `MODULE$` belong to the owning serializer and should use a cached `_JDKAccess._trustedLookup(...).findStaticGetter(...)` handle at that call site. - JDK25+ final-instance field mutation should use the same trusted-lookup `VarHandle` owner path as non-final field mutation. Do not add a final-field-only `MethodHandle` setter path or ordinary reflective `Field.set*` fallback. -- JDK25+ `FieldAccessorStrategy` should use one final trusted-lookup `VarHandle` instance accessor +- JDK25+ `InstanceFieldAccessors` should use one final trusted-lookup `VarHandle` instance accessor with dense access-kind switches, not public primitive/object accessor classes or a JDK25 `GeneratedAccessor` or hidden-class accessor generation. Do not wrap `VarHandle.get/set` in hot-path try/catch blocks and do not call `FieldAccessor.checkObj`; VarHandle validates null and diff --git a/benchmarks/java/pom.xml b/benchmarks/java/pom.xml index 812906d202..660b99e52e 100644 --- a/benchmarks/java/pom.xml +++ b/benchmarks/java/pom.xml @@ -268,7 +268,7 @@ + name="META-INF/versions/25/org/apache/fory/reflect/InstanceFieldAccessors.class"/> @@ -289,8 +289,8 @@ file="${jdk25.benchmark.check.dir}/META-INF/versions/25/org/apache/fory/platform/internal/_UnsafeUtils.class" property="jdk25.benchmark.unsafeutils.present"/> + file="${jdk25.benchmark.check.dir}/META-INF/versions/25/org/apache/fory/reflect/InstanceFieldAccessors.class" + property="jdk25.benchmark.instancefieldaccessors.present"/> @@ -310,8 +310,8 @@ unless="jdk25.benchmark.unsafeutils.present" message="JDK25 benchmark jar is missing the versioned _UnsafeUtils class."/> + unless="jdk25.benchmark.instancefieldaccessors.present" + message="JDK25 benchmark jar is missing the versioned InstanceFieldAccessors class."/> diff --git a/benchmarks/java/src/main/java/org/apache/fory/benchmark/Jdk25MrJarCheck.java b/benchmarks/java/src/main/java/org/apache/fory/benchmark/Jdk25MrJarCheck.java index 38ac95853b..8bdc70b66b 100644 --- a/benchmarks/java/src/main/java/org/apache/fory/benchmark/Jdk25MrJarCheck.java +++ b/benchmarks/java/src/main/java/org/apache/fory/benchmark/Jdk25MrJarCheck.java @@ -30,7 +30,7 @@ public static void main(String[] args) { verifyMissing("org.apache.fory.platform.UnsafeOps"); Class jdkAccess = verifyRootClass("org.apache.fory.platform.internal._JDKAccess"); Class unsafeUtils = verifyVersionedClass("org.apache.fory.platform.internal._UnsafeUtils"); - verifyVersionedClass("org.apache.fory.reflect.FieldAccessorStrategy"); + verifyVersionedClass("org.apache.fory.reflect.InstanceFieldAccessors"); verifyVersionedClass("org.apache.fory.serializer.PlatformStringUtils"); verifyNoUnsafeDescriptors(jdkAccess); verifyNoUnsafeDescriptors(unsafeUtils); diff --git a/java/fory-core/pom.xml b/java/fory-core/pom.xml index da608c772b..e40b680c7e 100644 --- a/java/fory-core/pom.xml +++ b/java/fory-core/pom.xml @@ -468,7 +468,7 @@ + name="META-INF/versions/25/org/apache/fory/reflect/InstanceFieldAccessors.java"/> @@ -484,8 +484,8 @@ file="${jdk25.sources.check.dir}/META-INF/versions/25/org/apache/fory/platform/internal/_UnsafeUtils.java" property="jdk25.unsafeutils.source.present"/> + file="${jdk25.sources.check.dir}/META-INF/versions/25/org/apache/fory/reflect/InstanceFieldAccessors.java" + property="jdk25.instancefieldaccessors.source.present"/> @@ -502,8 +502,8 @@ unless="jdk25.unsafeutils.source.present" message="JDK25 versioned _UnsafeUtils source is missing from the source jar."/> + unless="jdk25.instancefieldaccessors.source.present" + message="JDK25 versioned InstanceFieldAccessors source is missing from the source jar."/> diff --git a/java/fory-core/src/main/java/org/apache/fory/builder/BaseObjectCodecBuilder.java b/java/fory-core/src/main/java/org/apache/fory/builder/BaseObjectCodecBuilder.java index 78cc01953d..4c047b45a5 100644 --- a/java/fory-core/src/main/java/org/apache/fory/builder/BaseObjectCodecBuilder.java +++ b/java/fory-core/src/main/java/org/apache/fory/builder/BaseObjectCodecBuilder.java @@ -122,7 +122,6 @@ import org.apache.fory.meta.TypeExtMeta; import org.apache.fory.platform.GraalvmSupport; import org.apache.fory.reflect.ObjectCreator; -import org.apache.fory.reflect.ObjectCreators; import org.apache.fory.reflect.ReflectionUtils; import org.apache.fory.reflect.TypeRef; import org.apache.fory.resolver.ClassResolver; @@ -289,11 +288,10 @@ protected Expression getObjectCreator(Class type) { ObjectCreator.class, ctx.newName("objectCreator_" + type.getSimpleName()), () -> - new StaticInvoke( - ObjectCreators.class, + new Invoke( + typeResolverRef, "getObjectCreator", TypeRef.of(ObjectCreator.class), - typeResolverRef, getClassExpr(type))); } diff --git a/java/fory-core/src/main/java/org/apache/fory/reflect/FieldAccessor.java b/java/fory-core/src/main/java/org/apache/fory/reflect/FieldAccessor.java index 4fbbf54f88..6600f79396 100644 --- a/java/fory-core/src/main/java/org/apache/fory/reflect/FieldAccessor.java +++ b/java/fory-core/src/main/java/org/apache/fory/reflect/FieldAccessor.java @@ -20,7 +20,10 @@ package org.apache.fory.reflect; import java.lang.reflect.Field; +import java.lang.reflect.Modifier; +import org.apache.fory.platform.AndroidSupport; import org.apache.fory.util.Preconditions; +import org.apache.fory.util.record.RecordUtils; /** Field accessor for primitive types and object types. */ public abstract class FieldAccessor { @@ -211,6 +214,15 @@ public Object getGetter() { } public static FieldAccessor createAccessor(Field field) { - return FieldAccessorFactory.createAccessor(field); + Preconditions.checkArgument(!Modifier.isStatic(field.getModifiers()), field); + if (RecordUtils.isRecord(field.getDeclaringClass())) { + return RecordFieldAccessors.createAccessor(field); + } + if (AndroidSupport.IS_ANDROID) { + // Android field access must stay reflection-owned: no Unsafe offsets, trusted lookups, + // generated accessors, or primitive-specific reflection subclasses. + return new ReflectionFieldAccessor(field); + } + return InstanceFieldAccessors.createAccessor(field); } } diff --git a/java/fory-core/src/main/java/org/apache/fory/reflect/FieldAccessorStrategy.java b/java/fory-core/src/main/java/org/apache/fory/reflect/InstanceFieldAccessors.java similarity index 99% rename from java/fory-core/src/main/java/org/apache/fory/reflect/FieldAccessorStrategy.java rename to java/fory-core/src/main/java/org/apache/fory/reflect/InstanceFieldAccessors.java index 9ac559fe6b..afa1505287 100644 --- a/java/fory-core/src/main/java/org/apache/fory/reflect/FieldAccessorStrategy.java +++ b/java/fory-core/src/main/java/org/apache/fory/reflect/InstanceFieldAccessors.java @@ -34,7 +34,7 @@ import org.apache.fory.util.Preconditions; import sun.misc.Unsafe; -final class FieldAccessorStrategy { +final class InstanceFieldAccessors { private static final Unsafe UNSAFE = AndroidSupport.IS_ANDROID ? null : _UnsafeUtils.UNSAFE; private static final int BOOLEAN_ACCESS = 1; @@ -47,7 +47,7 @@ final class FieldAccessorStrategy { private static final int DOUBLE_ACCESS = 8; private static final int OBJECT_ACCESS = 9; - private FieldAccessorStrategy() {} + private InstanceFieldAccessors() {} static FieldAccessor createAccessor(Field field) { Preconditions.checkArgument(!Modifier.isStatic(field.getModifiers()), field); diff --git a/java/fory-core/src/main/java/org/apache/fory/reflect/FieldAccessorFactory.java b/java/fory-core/src/main/java/org/apache/fory/reflect/RecordFieldAccessors.java similarity index 91% rename from java/fory-core/src/main/java/org/apache/fory/reflect/FieldAccessorFactory.java rename to java/fory-core/src/main/java/org/apache/fory/reflect/RecordFieldAccessors.java index a9d5302482..bc841a9c4a 100644 --- a/java/fory-core/src/main/java/org/apache/fory/reflect/FieldAccessorFactory.java +++ b/java/fory-core/src/main/java/org/apache/fory/reflect/RecordFieldAccessors.java @@ -22,7 +22,6 @@ import java.lang.reflect.Field; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; -import java.lang.reflect.Modifier; import java.util.function.Function; import java.util.function.Predicate; import java.util.function.ToDoubleFunction; @@ -38,26 +37,15 @@ import org.apache.fory.util.function.ToCharFunction; import org.apache.fory.util.function.ToFloatFunction; import org.apache.fory.util.function.ToShortFunction; -import org.apache.fory.util.record.RecordUtils; -final class FieldAccessorFactory { - private FieldAccessorFactory() {} +final class RecordFieldAccessors { + private RecordFieldAccessors() {} static FieldAccessor createAccessor(Field field) { - Preconditions.checkArgument(!Modifier.isStatic(field.getModifiers()), field); - if (RecordUtils.isRecord(field.getDeclaringClass())) { - return createRecordAccessor(field); - } if (AndroidSupport.IS_ANDROID) { - // Android field access must stay reflection-owned: no Unsafe offsets, trusted lookups, - // generated accessors, or primitive-specific reflection subclasses. - return new ReflectionFieldAccessor(field); + return new ReflectiveRecordFieldAccessor(field); } - return FieldAccessorStrategy.createAccessor(field); - } - - private static FieldAccessor createRecordAccessor(Field field) { - if (AndroidSupport.IS_ANDROID || GraalvmSupport.IN_GRAALVM_NATIVE_IMAGE) { + if (GraalvmSupport.IN_GRAALVM_NATIVE_IMAGE) { return new ReflectiveRecordFieldAccessor(field); } Object getter; diff --git a/java/fory-core/src/main/java25/org/apache/fory/reflect/FieldAccessorStrategy.java b/java/fory-core/src/main/java25/org/apache/fory/reflect/InstanceFieldAccessors.java similarity index 99% rename from java/fory-core/src/main/java25/org/apache/fory/reflect/FieldAccessorStrategy.java rename to java/fory-core/src/main/java25/org/apache/fory/reflect/InstanceFieldAccessors.java index 97376a2471..4982ca64ac 100644 --- a/java/fory-core/src/main/java25/org/apache/fory/reflect/FieldAccessorStrategy.java +++ b/java/fory-core/src/main/java25/org/apache/fory/reflect/InstanceFieldAccessors.java @@ -25,7 +25,7 @@ import org.apache.fory.platform.internal._JDKAccess; import org.apache.fory.util.Preconditions; -final class FieldAccessorStrategy { +final class InstanceFieldAccessors { private static final int BOOLEAN_ACCESS = 1; private static final int BYTE_ACCESS = 2; private static final int CHAR_ACCESS = 3; @@ -36,7 +36,7 @@ final class FieldAccessorStrategy { private static final int DOUBLE_ACCESS = 8; private static final int OBJECT_ACCESS = 9; - private FieldAccessorStrategy() {} + private InstanceFieldAccessors() {} static FieldAccessor createAccessor(Field field) { Preconditions.checkArgument(!Modifier.isStatic(field.getModifiers()), field); diff --git a/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties b/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties index d0586147a2..498c002bab 100644 --- a/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties +++ b/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties @@ -248,8 +248,8 @@ Args=--initialize-at-build-time=org.apache.fory.annotation.ForyField$Dynamic,\ org.apache.fory.meta.MetaStringEncoder,\ org.apache.fory.meta.TypeEqualMetaCompressor,\ org.apache.fory.pool.ThreadPoolFory,\ - org.apache.fory.reflect.FieldAccessorStrategy,\ - org.apache.fory.reflect.FieldAccessorStrategy$GeneratedAccessor,\ + org.apache.fory.reflect.InstanceFieldAccessors,\ + org.apache.fory.reflect.InstanceFieldAccessors$GeneratedAccessor,\ org.apache.fory.reflect.ReflectionUtils$1,\ org.apache.fory.reflect.ReflectionUtils$2,\ org.apache.fory.reflect.ReflectionUtils,\ diff --git a/java/fory-core/src/test/java/org/apache/fory/builder/Jdk25MultiReleaseJarVerifier.java b/java/fory-core/src/test/java/org/apache/fory/builder/Jdk25MultiReleaseJarVerifier.java index e27763be3d..d79b956efc 100644 --- a/java/fory-core/src/test/java/org/apache/fory/builder/Jdk25MultiReleaseJarVerifier.java +++ b/java/fory-core/src/test/java/org/apache/fory/builder/Jdk25MultiReleaseJarVerifier.java @@ -51,7 +51,7 @@ public final class Jdk25MultiReleaseJarVerifier { "org/apache/fory/platform/internal/_Lookup.class", "org/apache/fory/platform/internal/_UnsafeUtils.class", "org/apache/fory/builder/UnsafeCodegenSupport.class", - "org/apache/fory/reflect/FieldAccessorStrategy.class", + "org/apache/fory/reflect/InstanceFieldAccessors.class", "org/apache/fory/reflect/ConstructorBypassAllocator.class", "org/apache/fory/serializer/PlatformStringUtils.class" }; diff --git a/java/fory-core/src/test/java/org/apache/fory/reflect/FieldAccessorTest.java b/java/fory-core/src/test/java/org/apache/fory/reflect/FieldAccessorTest.java index e3daac37d0..98aeb6d332 100644 --- a/java/fory-core/src/test/java/org/apache/fory/reflect/FieldAccessorTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/reflect/FieldAccessorTest.java @@ -27,7 +27,7 @@ import lombok.AllArgsConstructor; import org.apache.fory.TestUtils; import org.apache.fory.platform.AndroidSupport; -import org.apache.fory.reflect.FieldAccessorStrategy.GeneratedAccessor; +import org.apache.fory.reflect.InstanceFieldAccessors.GeneratedAccessor; import org.testng.Assert; import org.testng.annotations.Test; From 0e78b32fd1333b476227afa7ccda041f3d2de86f Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Tue, 2 Jun 2026 18:24:52 +0800 Subject: [PATCH 54/69] refactor(java): fold android field reflection into instance accessors --- .agents/languages/java.md | 5 ++ .../apache/fory/reflect/FieldAccessor.java | 6 --- .../fory/reflect/InstanceFieldAccessors.java | 52 ++++++++++++++----- .../fory/reflect/ReflectionFieldAccessor.java | 52 ------------------- .../fory/reflect/FieldAccessorTest.java | 3 +- 5 files changed, 46 insertions(+), 72 deletions(-) delete mode 100644 java/fory-core/src/main/java/org/apache/fory/reflect/ReflectionFieldAccessor.java diff --git a/.agents/languages/java.md b/.agents/languages/java.md index f3b631a6a7..7803f25fcc 100644 --- a/.agents/languages/java.md +++ b/.agents/languages/java.md @@ -116,6 +116,11 @@ Load this file when changing anything under `java/` or when Java drives a cross- - `FieldAccessor` owns field-accessor dispatch. `RecordFieldAccessors` owns record field access, and `InstanceFieldAccessors` owns non-record instance field access. Do not reintroduce a `FieldAccessorFactory` layer. +- Android non-record reflection field access belongs inside the root `InstanceFieldAccessors` + owner. Do not keep a standalone `ReflectionFieldAccessor`; Java25 never needs that path, and + record reflection fallback remains record-owned in `RecordFieldAccessors`. Keep `sun.misc.Unsafe` + fields and descriptors below the JVM-only nested instance accessor so Android can load the owner + and take the reflection branch. - JDK25+ `InstanceFieldAccessors` owns instance field access only. Do not add static-field handling or per-write reflection fallback there, and do not expose public `FieldAccessor` static-field factories. Static special cases such as Scala `MODULE$` belong to the owning serializer and should diff --git a/java/fory-core/src/main/java/org/apache/fory/reflect/FieldAccessor.java b/java/fory-core/src/main/java/org/apache/fory/reflect/FieldAccessor.java index 6600f79396..0eeff11605 100644 --- a/java/fory-core/src/main/java/org/apache/fory/reflect/FieldAccessor.java +++ b/java/fory-core/src/main/java/org/apache/fory/reflect/FieldAccessor.java @@ -21,7 +21,6 @@ import java.lang.reflect.Field; import java.lang.reflect.Modifier; -import org.apache.fory.platform.AndroidSupport; import org.apache.fory.util.Preconditions; import org.apache.fory.util.record.RecordUtils; @@ -218,11 +217,6 @@ public static FieldAccessor createAccessor(Field field) { if (RecordUtils.isRecord(field.getDeclaringClass())) { return RecordFieldAccessors.createAccessor(field); } - if (AndroidSupport.IS_ANDROID) { - // Android field access must stay reflection-owned: no Unsafe offsets, trusted lookups, - // generated accessors, or primitive-specific reflection subclasses. - return new ReflectionFieldAccessor(field); - } return InstanceFieldAccessors.createAccessor(field); } } diff --git a/java/fory-core/src/main/java/org/apache/fory/reflect/InstanceFieldAccessors.java b/java/fory-core/src/main/java/org/apache/fory/reflect/InstanceFieldAccessors.java index afa1505287..932d72ec18 100644 --- a/java/fory-core/src/main/java/org/apache/fory/reflect/InstanceFieldAccessors.java +++ b/java/fory-core/src/main/java/org/apache/fory/reflect/InstanceFieldAccessors.java @@ -27,6 +27,7 @@ import java.util.concurrent.ConcurrentMap; import org.apache.fory.collection.ClassValueCache; import org.apache.fory.collection.Tuple2; +import org.apache.fory.exception.ForyException; import org.apache.fory.platform.AndroidSupport; import org.apache.fory.platform.GraalvmSupport; import org.apache.fory.platform.internal._JDKAccess; @@ -35,8 +36,6 @@ import sun.misc.Unsafe; final class InstanceFieldAccessors { - private static final Unsafe UNSAFE = AndroidSupport.IS_ANDROID ? null : _UnsafeUtils.UNSAFE; - private static final int BOOLEAN_ACCESS = 1; private static final int BYTE_ACCESS = 2; private static final int CHAR_ACCESS = 3; @@ -51,21 +50,13 @@ private InstanceFieldAccessors() {} static FieldAccessor createAccessor(Field field) { Preconditions.checkArgument(!Modifier.isStatic(field.getModifiers()), field); - if (GraalvmSupport.isGraalBuildTime()) { - return new GeneratedAccessor(field); - } - return new InstanceAccessor(field); - } - - private static long fieldOffset(Field field) { if (AndroidSupport.IS_ANDROID) { - return -1; + return new ReflectionAccessor(field); } if (GraalvmSupport.isGraalBuildTime()) { - // Field offsets are rewritten by GraalVM and are not stable during native-image build time. - return -1; + return new GeneratedAccessor(field); } - return UNSAFE.objectFieldOffset(field); + return new InstanceAccessor(field); } private static int accessKind(Field field) { @@ -90,7 +81,38 @@ private static int accessKind(Field field) { return OBJECT_ACCESS; } + private static final class ReflectionAccessor extends FieldAccessor { + private ReflectionAccessor(Field field) { + super(field); + try { + field.setAccessible(true); + } catch (RuntimeException e) { + throw new ForyException("Failed to make field accessible: " + field, e); + } + } + + @Override + public Object get(Object obj) { + try { + return field.get(obj); + } catch (IllegalAccessException | IllegalArgumentException e) { + throw new ForyException("Failed to read field reflectively: " + field, e); + } + } + + @Override + public void set(Object obj, Object value) { + try { + field.set(obj, value); + } catch (IllegalAccessException | IllegalArgumentException e) { + throw new ForyException("Failed to write field reflectively: " + field, e); + } + } + } + static final class InstanceAccessor extends FieldAccessor { + private static final Unsafe UNSAFE = _UnsafeUtils.UNSAFE; + private final long fieldOffset; private final int accessKind; @@ -100,6 +122,10 @@ static final class InstanceAccessor extends FieldAccessor { accessKind = accessKind(field); } + private static long fieldOffset(Field field) { + return UNSAFE.objectFieldOffset(field); + } + @Override public Object get(Object obj) { checkObj(obj); diff --git a/java/fory-core/src/main/java/org/apache/fory/reflect/ReflectionFieldAccessor.java b/java/fory-core/src/main/java/org/apache/fory/reflect/ReflectionFieldAccessor.java deleted file mode 100644 index 25e43a929e..0000000000 --- a/java/fory-core/src/main/java/org/apache/fory/reflect/ReflectionFieldAccessor.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.fory.reflect; - -import java.lang.reflect.Field; -import org.apache.fory.exception.ForyException; - -final class ReflectionFieldAccessor extends FieldAccessor { - ReflectionFieldAccessor(Field field) { - super(field); - try { - field.setAccessible(true); - } catch (RuntimeException e) { - throw new ForyException("Failed to make field accessible: " + field, e); - } - } - - @Override - public Object get(Object obj) { - try { - return field.get(obj); - } catch (IllegalAccessException | IllegalArgumentException e) { - throw new ForyException("Failed to read field reflectively: " + field, e); - } - } - - @Override - public void set(Object obj, Object value) { - try { - field.set(obj, value); - } catch (IllegalAccessException | IllegalArgumentException e) { - throw new ForyException("Failed to write field reflectively: " + field, e); - } - } -} diff --git a/java/fory-core/src/test/java/org/apache/fory/reflect/FieldAccessorTest.java b/java/fory-core/src/test/java/org/apache/fory/reflect/FieldAccessorTest.java index 98aeb6d332..14778e4d94 100644 --- a/java/fory-core/src/test/java/org/apache/fory/reflect/FieldAccessorTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/reflect/FieldAccessorTest.java @@ -143,7 +143,8 @@ private static void assertAccessor( Field field = AndroidFields.class.getDeclaredField(fieldName); FieldAccessor accessor = FieldAccessor.createAccessor(field); check( - accessor instanceof ReflectionFieldAccessor, "Expected reflection accessor for " + field); + accessor.getClass().getEnclosingClass() == InstanceFieldAccessors.class, + "Expected instance accessor owner for " + field); checkEquals(accessor.get(fields), expected, "initial " + fieldName); accessor.set(fields, replacement); checkEquals(accessor.get(fields), replacement, "updated " + fieldName); From 8e5a3bfd6ce2f51e38fd4ca863617b1e4ef1166e Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Tue, 2 Jun 2026 18:47:27 +0800 Subject: [PATCH 55/69] perf(java): specialize jdk25 generated field accessors --- .agents/languages/java.md | 12 ++- .../org/apache/fory/builder/CodecBuilder.java | 19 ++++ .../fory/reflect/InstanceFieldAccessors.java | 14 ++- .../fory/reflect/InstanceFieldAccessors.java | 14 ++- .../Jdk25GeneratedFieldAccessorTest.java | 92 +++++++++++++++++++ 5 files changed, 146 insertions(+), 5 deletions(-) create mode 100644 java/fory-latest-jdk-tests/src/test/java/org/apache/fory/integration_tests/Jdk25GeneratedFieldAccessorTest.java diff --git a/.agents/languages/java.md b/.agents/languages/java.md index 7803f25fcc..ae395d1bf6 100644 --- a/.agents/languages/java.md +++ b/.agents/languages/java.md @@ -115,7 +115,8 @@ Load this file when changing anything under `java/` or when Java drives a cross- trusted-lookup initialization or cold setup, not inside string hot paths. - `FieldAccessor` owns field-accessor dispatch. `RecordFieldAccessors` owns record field access, and `InstanceFieldAccessors` owns non-record instance field access. Do not reintroduce a - `FieldAccessorFactory` layer. + `FieldAccessorFactory` layer. `InstanceFieldAccessors` is public only so generated serializers + can name its concrete nested accessor type; treat it as internal owner code, not user API. - Android non-record reflection field access belongs inside the root `InstanceFieldAccessors` owner. Do not keep a standalone `ReflectionFieldAccessor`; Java25 never needs that path, and record reflection fallback remains record-owned in `RecordFieldAccessors`. Keep `sun.misc.Unsafe` @@ -134,6 +135,15 @@ Load this file when changing anything under `java/` or when Java drives a cross- hot-path try/catch blocks and do not call `FieldAccessor.checkObj`; VarHandle validates null and receiver type itself. Root Unsafe offset access may keep a debug-only `assert` receiver check because Unsafe does not validate the target object; do not add production receiver checks. +- JDK25+ generated serializers should store field accessors as concrete + `InstanceFieldAccessors.InstanceAccessor` static final fields, initialized once through + `FieldAccessor.createAccessor(...)` and a static-init cast. This keeps platform dispatch out of + generated read/write hot paths and avoids `FieldAccessor` virtual dispatch on final/private field + get/set calls. +- Do not move the JDK25 `DefineClass.defineHiddenNestmate(...)` implementation to `java9`. + `Lookup#defineClass` is a Java 9 normal class-definition API, not hidden-nestmate definition; the + dynamic nestmate API is `Lookup#defineHiddenClass(..., NESTMATE)`, so Java 9 through Java 14 + cannot use that path. - Keep JDK26 `--illegal-final-field-mutation=deny` scoped to the JPMS runtime tests that prove Fory's field-access path. Do not put it in global Maven `JDK_JAVA_OPTIONS`, because build tools such as Lombok may perform their own reflective final-field access during compilation. diff --git a/java/fory-core/src/main/java/org/apache/fory/builder/CodecBuilder.java b/java/fory-core/src/main/java/org/apache/fory/builder/CodecBuilder.java index c52d771e80..deaab56fc4 100644 --- a/java/fory-core/src/main/java/org/apache/fory/builder/CodecBuilder.java +++ b/java/fory-core/src/main/java/org/apache/fory/builder/CodecBuilder.java @@ -59,6 +59,7 @@ import org.apache.fory.platform.GraalvmSupport; import org.apache.fory.platform.JdkVersion; import org.apache.fory.reflect.FieldAccessor; +import org.apache.fory.reflect.InstanceFieldAccessors.InstanceAccessor; import org.apache.fory.reflect.ObjectCreator; import org.apache.fory.reflect.ObjectCreators; import org.apache.fory.reflect.ReflectionUtils; @@ -425,6 +426,24 @@ private Reference getFieldAccessor(Descriptor descriptor) { : "") + fieldName + "_accessor_"; + if (JdkVersion.MAJOR_VERSION >= 25) { + // JDK25+ field writes go through the VarHandle-backed instance accessor. Keep the generated + // static field typed as the concrete final accessor so hot-path putX calls do not pay a + // FieldAccessor virtual dispatch. FieldAccessor.createAccessor still owns platform dispatch; + // this one-time cast happens only during generated-class initialization. + return getOrCreateField( + true, + InstanceAccessor.class, + fieldAccessorName, + () -> + new Cast( + new StaticInvoke( + FieldAccessor.class, + "createAccessor", + TypeRef.of(FieldAccessor.class), + getReflectField(field.getDeclaringClass(), field, false)), + TypeRef.of(InstanceAccessor.class))); + } return getOrCreateField( true, FieldAccessor.class, diff --git a/java/fory-core/src/main/java/org/apache/fory/reflect/InstanceFieldAccessors.java b/java/fory-core/src/main/java/org/apache/fory/reflect/InstanceFieldAccessors.java index 932d72ec18..cc664e32b7 100644 --- a/java/fory-core/src/main/java/org/apache/fory/reflect/InstanceFieldAccessors.java +++ b/java/fory-core/src/main/java/org/apache/fory/reflect/InstanceFieldAccessors.java @@ -25,6 +25,7 @@ import java.lang.reflect.Modifier; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +import org.apache.fory.annotation.Internal; import org.apache.fory.collection.ClassValueCache; import org.apache.fory.collection.Tuple2; import org.apache.fory.exception.ForyException; @@ -35,7 +36,15 @@ import org.apache.fory.util.Preconditions; import sun.misc.Unsafe; -final class InstanceFieldAccessors { +/** + * Non-record instance field accessor owner. + * + *

    This class is public only so generated serializers can name {@link InstanceAccessor} as a + * concrete field type on JDK25+. Callers must still create accessors through {@link + * FieldAccessor#createAccessor(Field)} so platform dispatch stays centralized. + */ +@Internal +public final class InstanceFieldAccessors { private static final int BOOLEAN_ACCESS = 1; private static final int BYTE_ACCESS = 2; private static final int CHAR_ACCESS = 3; @@ -110,7 +119,8 @@ public void set(Object obj, Object value) { } } - static final class InstanceAccessor extends FieldAccessor { + /** Public only for generated serializers; use {@link FieldAccessor#createAccessor(Field)}. */ + public static final class InstanceAccessor extends FieldAccessor { private static final Unsafe UNSAFE = _UnsafeUtils.UNSAFE; private final long fieldOffset; diff --git a/java/fory-core/src/main/java25/org/apache/fory/reflect/InstanceFieldAccessors.java b/java/fory-core/src/main/java25/org/apache/fory/reflect/InstanceFieldAccessors.java index 4982ca64ac..7963b1b221 100644 --- a/java/fory-core/src/main/java25/org/apache/fory/reflect/InstanceFieldAccessors.java +++ b/java/fory-core/src/main/java25/org/apache/fory/reflect/InstanceFieldAccessors.java @@ -22,10 +22,19 @@ import java.lang.invoke.VarHandle; import java.lang.reflect.Field; import java.lang.reflect.Modifier; +import org.apache.fory.annotation.Internal; import org.apache.fory.platform.internal._JDKAccess; import org.apache.fory.util.Preconditions; -final class InstanceFieldAccessors { +/** + * Non-record instance field accessor owner. + * + *

    This class is public only so generated serializers can name {@link InstanceAccessor} as a + * concrete field type on JDK25+. Callers must still create accessors through {@link + * FieldAccessor#createAccessor(Field)} so platform dispatch stays centralized. + */ +@Internal +public final class InstanceFieldAccessors { private static final int BOOLEAN_ACCESS = 1; private static final int BYTE_ACCESS = 2; private static final int CHAR_ACCESS = 3; @@ -86,7 +95,8 @@ private static IllegalStateException accessFailure(Field field, Throwable cause) cause); } - static final class InstanceAccessor extends FieldAccessor { + /** Public only for generated serializers; use {@link FieldAccessor#createAccessor(Field)}. */ + public static final class InstanceAccessor extends FieldAccessor { private final VarHandle handle; private final int accessKind; diff --git a/java/fory-latest-jdk-tests/src/test/java/org/apache/fory/integration_tests/Jdk25GeneratedFieldAccessorTest.java b/java/fory-latest-jdk-tests/src/test/java/org/apache/fory/integration_tests/Jdk25GeneratedFieldAccessorTest.java new file mode 100644 index 0000000000..e69223878d --- /dev/null +++ b/java/fory-latest-jdk-tests/src/test/java/org/apache/fory/integration_tests/Jdk25GeneratedFieldAccessorTest.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.integration_tests; + +import java.lang.reflect.Field; +import org.apache.fory.Fory; +import org.apache.fory.serializer.CodegenSerializer.LazyInitBeanSerializer; +import org.apache.fory.serializer.Serializer; +import org.testng.Assert; +import org.testng.annotations.Test; + +public class Jdk25GeneratedFieldAccessorTest { + private static final String INSTANCE_ACCESSOR = + "org.apache.fory.reflect.InstanceFieldAccessors$InstanceAccessor"; + + @Test + public void testConcreteAccessorField() { + Fory fory = + Fory.builder() + .withXlang(false) + .withCodegen(true) + .withRefTracking(false) + .requireClassRegistration(false) + .build(); + PrivateFinalBean value = new PrivateFinalBean(7, "name"); + + PrivateFinalBean copy = (PrivateFinalBean) fory.deserialize(fory.serialize(value)); + + Assert.assertEquals(copy.number(), 7); + Assert.assertEquals(copy.name(), "name"); + Class serializerClass = serializerClass(fory, PrivateFinalBean.class); + assertAccessorField(serializerClass, "number"); + assertAccessorField(serializerClass, "name"); + } + + private static Class serializerClass(Fory fory, Class type) { + Serializer serializer = fory.getTypeResolver().getSerializer(type); + if (serializer instanceof LazyInitBeanSerializer) { + return ((LazyInitBeanSerializer) serializer).getGeneratedSerializerClass(); + } + return serializer.getClass(); + } + + private static void assertAccessorField(Class serializerClass, String fieldName) { + for (Field field : serializerClass.getDeclaredFields()) { + if (field.getName().contains(fieldName + "_accessor_")) { + Assert.assertEquals(field.getType().getName(), INSTANCE_ACCESSOR); + return; + } + } + Assert.fail("Missing generated accessor field for " + fieldName + " in " + serializerClass); + } + + public static final class PrivateFinalBean { + private final int number; + private final String name; + + public PrivateFinalBean() { + this(0, null); + } + + public PrivateFinalBean(int number, String name) { + this.number = number; + this.name = name; + } + + public int number() { + return number; + } + + public String name() { + return name; + } + } +} From b47856d5e5a4154b2a4fb0a4a3ee4677a5eb262b Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Tue, 2 Jun 2026 18:57:24 +0800 Subject: [PATCH 56/69] refactor(java): root-own hidden nestmate definition --- .agents/languages/java.md | 11 ++- .../fory/platform/internal/DefineClass.java | 83 ++++++++++++++++- .../fory/platform/internal/DefineClass.java | 93 ------------------- .../builder/Jdk25MultiReleaseJarVerifier.java | 1 - .../platform/internal/DefineClassTest.java | 28 ++++++ 5 files changed, 115 insertions(+), 101 deletions(-) delete mode 100644 java/fory-core/src/main/java25/org/apache/fory/platform/internal/DefineClass.java diff --git a/.agents/languages/java.md b/.agents/languages/java.md index ae395d1bf6..acb844162f 100644 --- a/.agents/languages/java.md +++ b/.agents/languages/java.md @@ -65,7 +65,7 @@ Load this file when changing anything under `java/` or when Java drives a cross- - JDK25+ zero-Unsafe final-field writes must use a true target-class trusted lookup from the original `IMPL_LOOKUP`, not `IMPL_LOOKUP.in(type)`. JDK26+ normal Fory final-field restoration must pass with `--illegal-final-field-mutation=deny` and must not require `--enable-final-field-mutation`. - For JDK25+ object creation, do not use `sun.reflect.ReflectionFactory` or `jdk.internal.reflect.ReflectionFactory`. The shared `ObjectCreators` facade should route ObjectStream-compatible construction through `ParentNoArgCtrObjectCreator`; the replaceable constructor-bypass allocator owns the JDK8-24 Unsafe allocation path and the JDK25+ `ObjectStreamClass.newInstance` path. Serializable classes without a no-arg constructor may use that trusted-lookup owner; non-Serializable classes without a no-arg constructor require `@ForyConstructor`, `BaseFory.registerConstructor(...)`, record construction, or a custom serializer. - In JDK25+ constructor-bypass allocation, the allocator is per type: pass the target class to the allocator constructor, cache `ObjectStreamClass.lookupAny(...)` in that instance, and expose an instance `allocate()` method. Let `ObjectStreamClass.newInstance` validate unsupported classes. Do not add redundant `Serializable` prechecks or per-call descriptor lookups, and keep exception/message construction in cold helper methods. -- Keep the Java25 `_Lookup` and `DefineClass` overlays unless a future refactor can merge them without exposing Unsafe to the JDK25 class graph or replacing direct hidden-class APIs with reflective wrappers. Root `_Lookup` uses Unsafe for the JDK8-24 trusted-lookup fast path, while Java25 `_Lookup` uses the required `java.lang.invoke` open. Root `DefineClass` targets Java 8 bytecode and cannot directly reference `Lookup#defineHiddenClass` or `Lookup.ClassOption.NESTMATE`; Java25 `DefineClass` owns that direct API use. +- Keep the Java25 `_Lookup` overlay unless a future refactor can merge it without exposing Unsafe to the JDK25 class graph. Root `_Lookup` uses Unsafe for the JDK8-24 trusted-lookup fast path, while Java25 `_Lookup` uses the required `java.lang.invoke` open. `DefineClass` is root-owned; when it needs hidden nestmate class definition, it must use cached method handles and reflective `Lookup.ClassOption.NESTMATE` loading so Java 8 through Java 14 can still load the root class safely. - Treat `ByteArrayOutputStream` and `ByteArrayInputStream` as ordinary streams on every JDK. Do not restore private-buffer wrapping for JDK8-24 performance, because that reintroduces `java.base/java.io` private-field ownership and module-open requirements. @@ -140,10 +140,11 @@ Load this file when changing anything under `java/` or when Java drives a cross- `FieldAccessor.createAccessor(...)` and a static-init cast. This keeps platform dispatch out of generated read/write hot paths and avoids `FieldAccessor` virtual dispatch on final/private field get/set calls. -- Do not move the JDK25 `DefineClass.defineHiddenNestmate(...)` implementation to `java9`. - `Lookup#defineClass` is a Java 9 normal class-definition API, not hidden-nestmate definition; the - dynamic nestmate API is `Lookup#defineHiddenClass(..., NESTMATE)`, so Java 9 through Java 14 - cannot use that path. +- `DefineClass#defineHiddenNestmate` belongs in the root `DefineClass` owner. Do not add a Java25 + overlay only to call `Lookup#defineHiddenClass` directly, and do not move it to `java9` because + `Lookup#defineClass` defines normal package classes, not hidden nestmates. Root code must avoid + direct `Lookup.ClassOption` linkage and cache the method-handle/option-array setup off the hot + path. - Keep JDK26 `--illegal-final-field-mutation=deny` scoped to the JPMS runtime tests that prove Fory's field-access path. Do not put it in global Maven `JDK_JAVA_OPTIONS`, because build tools such as Lombok may perform their own reflective final-field access during compilation. diff --git a/java/fory-core/src/main/java/org/apache/fory/platform/internal/DefineClass.java b/java/fory-core/src/main/java/org/apache/fory/platform/internal/DefineClass.java index c340e673a7..d465038815 100644 --- a/java/fory-core/src/main/java/org/apache/fory/platform/internal/DefineClass.java +++ b/java/fory-core/src/main/java/org/apache/fory/platform/internal/DefineClass.java @@ -21,7 +21,9 @@ import java.lang.invoke.MethodHandle; import java.lang.invoke.MethodHandles; +import java.lang.invoke.MethodHandles.Lookup; import java.lang.invoke.MethodType; +import java.lang.reflect.Array; import java.security.ProtectionDomain; import org.apache.fory.annotation.Internal; import org.apache.fory.platform.AndroidSupport; @@ -32,6 +34,7 @@ @Internal public class DefineClass { private static volatile MethodHandle classloaderDefineClassHandle; + private static volatile HiddenClassDefiner hiddenClassDefiner; public static Class defineClass( String className, @@ -80,7 +83,83 @@ public static Class defineClass( } public static Class defineHiddenNestmate(Class neighbor, byte[] bytecodes) { - throw new UnsupportedOperationException( - "Hidden nestmate class definition requires the JDK25 multi-release DefineClass"); + Preconditions.checkNotNull(neighbor); + Preconditions.checkNotNull(bytecodes); + if (JdkVersion.MAJOR_VERSION < 15) { + throw hiddenClassUnsupported(null); + } + try { + Lookup lookup = _Lookup.privateLookupIn(neighbor, MethodHandles.lookup()); + return hiddenClassDefiner().define(lookup, bytecodes); + } catch (Throwable e) { + throw hiddenClassFailure(neighbor, e); + } + } + + private static HiddenClassDefiner hiddenClassDefiner() { + HiddenClassDefiner definer = hiddenClassDefiner; + if (definer == null) { + synchronized (DefineClass.class) { + definer = hiddenClassDefiner; + if (definer == null) { + definer = loadHiddenClassDefiner(); + hiddenClassDefiner = definer; + } + } + } + return definer; + } + + private static HiddenClassDefiner loadHiddenClassDefiner() { + try { + // Keep JDK15+ hidden-class symbols out of the root constant pool so Java 8 through Java 14 + // can load this class; defineHiddenNestmate is the only path that needs them. + Class optionType = Class.forName("java.lang.invoke.MethodHandles$Lookup$ClassOption"); + Object nestmate = Enum.valueOf(optionType.asSubclass(Enum.class), "NESTMATE"); + Object options = Array.newInstance(optionType, 1); + Array.set(options, 0, nestmate); + MethodHandle defineHiddenClass = + MethodHandles.publicLookup() + .findVirtual( + Lookup.class, + "defineHiddenClass", + MethodType.methodType( + Lookup.class, byte[].class, boolean.class, options.getClass())) + .asFixedArity(); + return new HiddenClassDefiner(defineHiddenClass, options); + } catch (ReflectiveOperationException | RuntimeException e) { + throw hiddenClassUnsupported(e); + } + } + + private static UnsupportedOperationException hiddenClassUnsupported(Throwable cause) { + return new UnsupportedOperationException( + "Hidden nestmate class definition requires JDK15+ Lookup#defineHiddenClass", cause); + } + + private static RuntimeException hiddenClassFailure(Class neighbor, Throwable cause) { + if (cause instanceof RuntimeException) { + return (RuntimeException) cause; + } + if (cause instanceof Error) { + throw (Error) cause; + } + return new IllegalStateException( + "Cannot define hidden nestmate for " + neighbor.getName() + ".", cause); + } + + private static final class HiddenClassDefiner { + private final MethodHandle defineHiddenClass; + private final Object nestmateOptions; + + private HiddenClassDefiner(MethodHandle defineHiddenClass, Object nestmateOptions) { + this.defineHiddenClass = defineHiddenClass; + this.nestmateOptions = nestmateOptions; + } + + private Class define(Lookup lookup, byte[] bytecodes) throws Throwable { + return ((Lookup) defineHiddenClass.invoke(lookup, bytecodes, true, nestmateOptions)) + .lookupClass(); + } } } diff --git a/java/fory-core/src/main/java25/org/apache/fory/platform/internal/DefineClass.java b/java/fory-core/src/main/java25/org/apache/fory/platform/internal/DefineClass.java deleted file mode 100644 index 5214658cfc..0000000000 --- a/java/fory-core/src/main/java25/org/apache/fory/platform/internal/DefineClass.java +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.fory.platform.internal; - -import java.lang.invoke.MethodHandle; -import java.lang.invoke.MethodHandles; -import java.lang.invoke.MethodHandles.Lookup; -import java.lang.invoke.MethodType; -import java.security.ProtectionDomain; -import org.apache.fory.annotation.Internal; -import org.apache.fory.platform.JdkVersion; -import org.apache.fory.util.Preconditions; - -/** A class to define bytecode as a class. */ -@Internal -public class DefineClass { - private static volatile MethodHandle classloaderDefineClassHandle; - - public static Class defineClass( - String className, - Class neighbor, - ClassLoader loader, - ProtectionDomain domain, - byte[] bytecodes) { - Preconditions.checkNotNull(loader); - Preconditions.checkArgument(JdkVersion.MAJOR_VERSION >= 8); - if (neighbor != null && JdkVersion.MAJOR_VERSION >= 9) { - // Classes in bytecode must be in same package as lookup class. - MethodHandles.Lookup lookup = MethodHandles.lookup(); - _JDKAccess.addReads(_JDKAccess.getModule(DefineClass.class), _JDKAccess.getModule(neighbor)); - lookup = _Lookup.privateLookupIn(neighbor, lookup); - return _Lookup.defineClass(lookup, bytecodes); - } - if (classloaderDefineClassHandle == null) { - MethodHandles.Lookup lookup = _JDKAccess._trustedLookup(ClassLoader.class); - try { - classloaderDefineClassHandle = - lookup.findVirtual( - ClassLoader.class, - "defineClass", - MethodType.methodType( - Class.class, - String.class, - byte[].class, - int.class, - int.class, - ProtectionDomain.class)); - } catch (NoSuchMethodException | IllegalAccessException e) { - throw new RuntimeException(e); - } - } - try { - return (Class) - classloaderDefineClassHandle.invokeWithArguments( - loader, className, bytecodes, 0, bytecodes.length, domain); - } catch (Throwable e) { - throw new RuntimeException(e); - } - } - - public static Class defineHiddenNestmate(Class neighbor, byte[] bytecodes) { - Preconditions.checkNotNull(neighbor); - Preconditions.checkNotNull(bytecodes); - try { - Lookup lookup = _Lookup.privateLookupIn(neighbor, MethodHandles.lookup()); - return lookup.defineHiddenClass(bytecodes, true, Lookup.ClassOption.NESTMATE).lookupClass(); - } catch (IllegalAccessException | IllegalStateException e) { - throw new IllegalStateException( - "Cannot define hidden nestmate for " - + neighbor.getName() - + ". JDK25 zero-Unsafe mode requires java.base/java.lang.invoke to be open to " - + "org.apache.fory.core", - e); - } - } -} diff --git a/java/fory-core/src/test/java/org/apache/fory/builder/Jdk25MultiReleaseJarVerifier.java b/java/fory-core/src/test/java/org/apache/fory/builder/Jdk25MultiReleaseJarVerifier.java index d79b956efc..33b29a7fbb 100644 --- a/java/fory-core/src/test/java/org/apache/fory/builder/Jdk25MultiReleaseJarVerifier.java +++ b/java/fory-core/src/test/java/org/apache/fory/builder/Jdk25MultiReleaseJarVerifier.java @@ -47,7 +47,6 @@ public final class Jdk25MultiReleaseJarVerifier { "module-info.class", "org/apache/fory/memory/LittleEndian.class", "org/apache/fory/memory/MemoryBuffer.class", - "org/apache/fory/platform/internal/DefineClass.class", "org/apache/fory/platform/internal/_Lookup.class", "org/apache/fory/platform/internal/_UnsafeUtils.class", "org/apache/fory/builder/UnsafeCodegenSupport.class", diff --git a/java/fory-core/src/test/java/org/apache/fory/platform/internal/DefineClassTest.java b/java/fory-core/src/test/java/org/apache/fory/platform/internal/DefineClassTest.java index 3e05a00792..ec4c7da0f5 100644 --- a/java/fory-core/src/test/java/org/apache/fory/platform/internal/DefineClassTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/platform/internal/DefineClassTest.java @@ -69,4 +69,32 @@ public void testDefineClass() throws ClassNotFoundException { className, null, DefineClassTest.class.getClassLoader(), null, bytecodes); } } + + @Test + public void testDefineHiddenNestmate() throws Exception { + if (JdkVersion.MAJOR_VERSION < 15) { + return; + } + String pkg = DefineClassTest.class.getPackage().getName(); + CompileUnit unit = + new CompileUnit( + pkg, + "HiddenNestmateA", + ("package " + + pkg + + ";\n" + + "public class HiddenNestmateA {\n" + + " public static String hello() { return \"HIDDEN\"; }\n" + + "}")); + byte[] bytecodes = + JaninoUtils.toBytecode(Thread.currentThread().getContextClassLoader(), unit) + .values() + .iterator() + .next(); + + Class clz = DefineClass.defineHiddenNestmate(DefineClassTest.class, bytecodes); + + Assert.assertEquals(clz.getMethod("hello").invoke(null), "HIDDEN"); + Assert.assertEquals(Class.class.getMethod("getNestHost").invoke(clz), DefineClassTest.class); + } } From 81767b43d14a3ddd3be9ef4821dd68014c35934d Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Tue, 2 Jun 2026 19:33:06 +0800 Subject: [PATCH 57/69] feat(java): define JDK25 serializers as hidden nestmates --- .agents/languages/java.md | 10 ++- .../fory/builder/BaseObjectCodecBuilder.java | 16 ++++ .../org/apache/fory/builder/CodecBuilder.java | 4 +- .../org/apache/fory/builder/CodecUtils.java | 47 ++++++++--- .../apache/fory/codegen/CodeGenerator.java | 84 +++++++++++++++++-- .../apache/fory/codegen/CodegenContext.java | 19 ++++- .../org/apache/fory/codegen/CompileUnit.java | 18 ++++ .../fory/codegen/ExpressionOptimizer.java | 12 +-- .../fory/platform/internal/DefineClass.java | 6 +- .../apache/fory/builder/CodecUtilsTest.java | 15 ++++ 10 files changed, 199 insertions(+), 32 deletions(-) diff --git a/.agents/languages/java.md b/.agents/languages/java.md index acb844162f..e5c589a1e9 100644 --- a/.agents/languages/java.md +++ b/.agents/languages/java.md @@ -65,7 +65,7 @@ Load this file when changing anything under `java/` or when Java drives a cross- - JDK25+ zero-Unsafe final-field writes must use a true target-class trusted lookup from the original `IMPL_LOOKUP`, not `IMPL_LOOKUP.in(type)`. JDK26+ normal Fory final-field restoration must pass with `--illegal-final-field-mutation=deny` and must not require `--enable-final-field-mutation`. - For JDK25+ object creation, do not use `sun.reflect.ReflectionFactory` or `jdk.internal.reflect.ReflectionFactory`. The shared `ObjectCreators` facade should route ObjectStream-compatible construction through `ParentNoArgCtrObjectCreator`; the replaceable constructor-bypass allocator owns the JDK8-24 Unsafe allocation path and the JDK25+ `ObjectStreamClass.newInstance` path. Serializable classes without a no-arg constructor may use that trusted-lookup owner; non-Serializable classes without a no-arg constructor require `@ForyConstructor`, `BaseFory.registerConstructor(...)`, record construction, or a custom serializer. - In JDK25+ constructor-bypass allocation, the allocator is per type: pass the target class to the allocator constructor, cache `ObjectStreamClass.lookupAny(...)` in that instance, and expose an instance `allocate()` method. Let `ObjectStreamClass.newInstance` validate unsupported classes. Do not add redundant `Serializable` prechecks or per-call descriptor lookups, and keep exception/message construction in cold helper methods. -- Keep the Java25 `_Lookup` overlay unless a future refactor can merge it without exposing Unsafe to the JDK25 class graph. Root `_Lookup` uses Unsafe for the JDK8-24 trusted-lookup fast path, while Java25 `_Lookup` uses the required `java.lang.invoke` open. `DefineClass` is root-owned; when it needs hidden nestmate class definition, it must use cached method handles and reflective `Lookup.ClassOption.NESTMATE` loading so Java 8 through Java 14 can still load the root class safely. +- Keep the Java25 `_Lookup` overlay unless a future refactor can merge it without exposing Unsafe to the JDK25 class graph. Root `_Lookup` uses Unsafe for the JDK8-24 trusted-lookup fast path, while Java25 `_Lookup` uses the required `java.lang.invoke` open. `DefineClass` is root-owned; when Java25+ generated serializers need hidden nestmate class definition, it must use cached method handles and reflective `Lookup.ClassOption.NESTMATE` loading so Java 8 through Java 14 can still load the root class safely. - Treat `ByteArrayOutputStream` and `ByteArrayInputStream` as ordinary streams on every JDK. Do not restore private-buffer wrapping for JDK8-24 performance, because that reintroduces `java.base/java.io` private-field ownership and module-open requirements. @@ -145,6 +145,14 @@ Load this file when changing anything under `java/` or when Java drives a cross- `Lookup#defineClass` defines normal package classes, not hidden nestmates. Root code must avoid direct `Lookup.ClassOption` linkage and cache the method-handle/option-array setup off the hot path. +- Hidden generated serializers are Java25+ only. Do not broaden serializer hidden-class definition + to Java15-24, because those runtimes still use the unsafe-backed field/object path. Keep + `AccessorHelper` as the source-generated same-package helper; do not turn it into a bytecode + hidden-field owner unless a separate Java25-only design explicitly requires that. +- JDK25 hidden generated serializers must not emit private split helper methods. Janino lowers + private instance helpers to static bridge methods whose receiver parameter uses the original + binary class name, which fails hidden-class verification. Use non-private final split helpers on + that path. - Keep JDK26 `--illegal-final-field-mutation=deny` scoped to the JPMS runtime tests that prove Fory's field-access path. Do not put it in global Maven `JDK_JAVA_OPTIONS`, because build tools such as Lombok may perform their own reflective final-field access during compilation. diff --git a/java/fory-core/src/main/java/org/apache/fory/builder/BaseObjectCodecBuilder.java b/java/fory-core/src/main/java/org/apache/fory/builder/BaseObjectCodecBuilder.java index 4c047b45a5..4fed0ddec7 100644 --- a/java/fory-core/src/main/java/org/apache/fory/builder/BaseObjectCodecBuilder.java +++ b/java/fory-core/src/main/java/org/apache/fory/builder/BaseObjectCodecBuilder.java @@ -220,6 +220,10 @@ public BaseObjectCodecBuilder(TypeRef beanType, Fory fory, Class parentSer descriptorDispatchId = new HashMap<>(); } + void setSamePackageAccess(boolean samePackageAccess) { + ctx.setSamePackageAccess(samePackageAccess); + } + // Must be static to be shared across the whole process life. private static final Map> idGenerator = new ConcurrentHashMap<>(); @@ -962,6 +966,10 @@ private Expression getOrCreateSerializer(Class cls, boolean isField) { // TODO(chaokunyang) add jdk17+ unexported class check. // non-public class can't be accessed in generated class. serializerClass = Serializer.class; + } else if (isHiddenClass(serializerClass)) { + // Hidden generated serializers are represented by their Class object but cannot be named + // or loaded from Java source, so generated fields must use the stable Serializer type. + serializerClass = Serializer.class; } else { ClassLoader beanClassClassLoader = beanClass.getClassLoader(); if (beanClassClassLoader == null) { @@ -1025,6 +1033,14 @@ private Expression getOrCreateSerializer(Class cls, boolean isField) { return serializerRef; } + private static boolean isHiddenClass(Class cls) { + try { + return (Boolean) Class.class.getMethod("isHidden").invoke(cls); + } catch (ReflectiveOperationException | RuntimeException e) { + return false; + } + } + protected Expression getOrCreateStringSerializer() { return getOrCreateSerializer(String.class); } diff --git a/java/fory-core/src/main/java/org/apache/fory/builder/CodecBuilder.java b/java/fory-core/src/main/java/org/apache/fory/builder/CodecBuilder.java index deaab56fc4..40f1fedb98 100644 --- a/java/fory-core/src/main/java/org/apache/fory/builder/CodecBuilder.java +++ b/java/fory-core/src/main/java/org/apache/fory/builder/CodecBuilder.java @@ -216,7 +216,7 @@ protected Expression getFieldValue(Expression inputBeanExpr, Descriptor descript if (isRecord) { return getRecordFieldValue(inputBeanExpr, descriptor); } - if (duplicatedFields.contains(fieldName) || !Modifier.isPublic(beanClass.getModifiers())) { + if (duplicatedFields.contains(fieldName) || !sourcePublicAccessible(beanClass)) { return unsafeAccessField(inputBeanExpr, beanClass, descriptor); } if (!sourcePublicAccessible(rawType)) { @@ -473,7 +473,7 @@ protected Expression setFieldValue(Expression bean, Descriptor d, Expression val } if (!d.isFinalField() && Modifier.isPublic(d.getModifiers()) - && Modifier.isPublic(d.getRawType().getModifiers())) { + && sourcePublicAccessible(d.getRawType())) { if (!d.getRawType().isAssignableFrom(value.type().getRawType())) { value = tryInlineCast(value, d.getTypeRef()); } diff --git a/java/fory-core/src/main/java/org/apache/fory/builder/CodecUtils.java b/java/fory-core/src/main/java/org/apache/fory/builder/CodecUtils.java index c9092f6723..2361179ddb 100644 --- a/java/fory-core/src/main/java/org/apache/fory/builder/CodecUtils.java +++ b/java/fory-core/src/main/java/org/apache/fory/builder/CodecUtils.java @@ -19,7 +19,6 @@ package org.apache.fory.builder; -import java.util.Collections; import java.util.concurrent.Callable; import java.util.concurrent.ConcurrentHashMap; import org.apache.fory.Fory; @@ -27,7 +26,10 @@ import org.apache.fory.codegen.CompileUnit; import org.apache.fory.collection.Tuple3; import org.apache.fory.meta.TypeDef; +import org.apache.fory.platform.AndroidSupport; import org.apache.fory.platform.GraalvmSupport; +import org.apache.fory.platform.JdkVersion; +import org.apache.fory.reflect.ReflectionUtils; import org.apache.fory.reflect.TypeRef; import org.apache.fory.resolver.TypeResolver; import org.apache.fory.serializer.Serializer; @@ -118,12 +120,6 @@ public static Class loadOrGenCompatibleLayerCodecClass @SuppressWarnings("unchecked") static Class loadOrGenCodecClass( Class beanClass, Fory fory, BaseObjectCodecBuilder codecBuilder) { - // use genCodeFunc to avoid gen code repeatedly - CompileUnit compileUnit = - new CompileUnit( - CodeGenerator.getPackage(beanClass), - codecBuilder.codecClassName(beanClass), - codecBuilder::genCode); CodeGenerator codeGenerator; ClassLoader beanClassClassLoader = beanClass.getClassLoader() == null @@ -134,15 +130,40 @@ static Class loadOrGenCodecClass( } TypeResolver typeResolver = fory.getTypeResolver(); codeGenerator = getCodeGenerator(beanClassClassLoader, typeResolver); - ClassLoader classLoader = - codeGenerator.compile( - Collections.singletonList(compileUnit), compileState -> compileState.lock.lock()); - String className = codecBuilder.codecQualifiedClassName(beanClass); + Class neighborClass = codecNeighbor(beanClass, beanClassClassLoader); + codecBuilder.setSamePackageAccess(neighborClass != null); + // use genCodeFunc to avoid gen code repeatedly + CompileUnit compileUnit = + new CompileUnit( + CodeGenerator.getPackage(beanClass), + codecBuilder.codecClassName(beanClass), + codecBuilder::genCode, + neighborClass); + return (Class) + codeGenerator.compileAndLoad(compileUnit, compileState -> compileState.lock.lock()); + } + + private static Class codecNeighbor(Class beanClass, ClassLoader beanClassClassLoader) { + // Hidden generated serializers are only a JDK25+ path. JDK8-24 keeps the existing unsafe-backed + // field/object access strategy, so broadening this to Java15+ adds complexity without removing + // unsafe from those runtimes. + if (AndroidSupport.IS_ANDROID + || JdkVersion.MAJOR_VERSION < 25 + || beanClass.getClassLoader() == null) { + return null; + } + if (!CodeGenerator.getPackage(beanClass).equals(ReflectionUtils.getPackage(beanClass))) { + return null; + } try { - return (Class) classLoader.loadClass(className); + // A generated serializer defined in the bean loader must resolve Fory runtime classes there. + if (beanClassClassLoader.loadClass(Fory.class.getName()) == Fory.class) { + return beanClass; + } } catch (ClassNotFoundException e) { - throw new IllegalStateException("Impossible because we just compiled class", e); + // The composed-loader path remains the owner when the bean loader cannot see Fory directly. } + return null; } private static CodeGenerator getCodeGenerator( diff --git a/java/fory-core/src/main/java/org/apache/fory/codegen/CodeGenerator.java b/java/fory-core/src/main/java/org/apache/fory/codegen/CodeGenerator.java index 76b3a311f6..f63e4151c9 100644 --- a/java/fory-core/src/main/java/org/apache/fory/codegen/CodeGenerator.java +++ b/java/fory-core/src/main/java/org/apache/fory/codegen/CodeGenerator.java @@ -40,6 +40,7 @@ import org.apache.fory.logging.LoggerFactory; import org.apache.fory.platform.AndroidSupport; import org.apache.fory.platform.GraalvmSupport; +import org.apache.fory.platform.internal.DefineClass; import org.apache.fory.reflect.ReflectionUtils; import org.apache.fory.util.ClassLoaderUtils; import org.apache.fory.util.ClassLoaderUtils.ByteArrayClassLoader; @@ -135,6 +136,46 @@ public ClassLoader compile(List units, CompileCallback callback) { } parentClassLoader = classLoader; } + Map classes = compileToBytecode(compileUnits, parentClassLoader, callback); + return defineClasses(classes); + } + + public Class compileAndLoad(CompileUnit unit, CompileCallback callback) { + Class neighborClass = unit.getNeighborClass(); + if (neighborClass == null) { + ClassLoader loader = compile(java.util.Collections.singletonList(unit), callback); + try { + return loader.loadClass(unit.getQualifiedClassName()); + } catch (ClassNotFoundException e) { + throw new IllegalStateException("Impossible because we just compiled class", e); + } + } + checkRuntimeCodegenSupported(); + DefineState defineState = getDefineState(unit.getQualifiedClassName()); + if (defineState.definedClass != null) { + return defineState.definedClass; + } + synchronized (defineState.lock) { + if (defineState.definedClass != null) { + return defineState.definedClass; + } + ClassLoader parentClassLoader = getClassLoader(); + Map classes = + compileToBytecode(java.util.Collections.singletonList(unit), parentClassLoader, callback); + byte[] bytecodes = classes.get(classFilepath(unit)); + if (bytecodes == null) { + throw new IllegalStateException( + "Compiler did not produce bytecode for " + unit.getQualifiedClassName()); + } + Class definedClass = DefineClass.defineHiddenNestmate(neighborClass, bytecodes); + defineState.definedClass = definedClass; + defineState.defined = true; + return definedClass; + } + } + + private Map compileToBytecode( + List compileUnits, ClassLoader parentClassLoader, CompileCallback callback) { CompileState compileState = getCompileState(compileUnits); callback.lock(compileState); Map classes; @@ -150,12 +191,8 @@ public ClassLoader compile(List units, CompileCallback callback) { } finally { compileState.lock.unlock(); } - for (Map.Entry e : classes.entrySet()) { - String key = e.getKey(); - byte[] value = e.getValue(); - } } - return defineClasses(classes); + return classes; } /** @@ -305,6 +342,7 @@ private String getCompileLockKey(List toCompile) { private static class DefineState { final Object lock; volatile boolean defined; + volatile Class definedClass; private DefineState() { this.lock = new Object(); @@ -411,7 +449,7 @@ public static String classFilepath(String fullClassName) { } public static String fullClassName(CompileUnit unit) { - return unit.pkg + "." + unit.mainClassName; + return unit.getQualifiedClassName(); } public static String fullClassNameFromClassFilePath(String classFilePath) { @@ -526,10 +564,40 @@ private static boolean classSourcePublicAccessible(Class clz) { if (clz.isPrimitive()) { return true; } - if (!ReflectionUtils.isPublic(clz)) { + if (!sourcePkgLevelAccessible(clz)) { return false; } - return sourcePkgLevelAccessible(clz); + Class current = clz; + while (current != null) { + if (!ReflectionUtils.isPublic(current)) { + return false; + } + current = current.getEnclosingClass(); + } + return true; + } + + public static boolean sourceAccessibleFrom(Class clz, String pkg) { + if (sourcePublicAccessible(clz)) { + return true; + } + if (clz.isArray()) { + return sourceAccessibleFrom(clz.getComponentType(), pkg); + } + if (!sourcePkgLevelAccessible(clz)) { + return false; + } + if (!ReflectionUtils.getPackage(clz).equals(pkg)) { + return false; + } + Class current = clz; + while (current != null) { + if (ReflectionUtils.isPrivate(current)) { + return false; + } + current = current.getEnclosingClass(); + } + return true; } private static final Map, Boolean> sourcePkgLevelAccessible = diff --git a/java/fory-core/src/main/java/org/apache/fory/codegen/CodegenContext.java b/java/fory-core/src/main/java/org/apache/fory/codegen/CodegenContext.java index ccfed32371..779d7d9cf3 100644 --- a/java/fory-core/src/main/java/org/apache/fory/codegen/CodegenContext.java +++ b/java/fory-core/src/main/java/org/apache/fory/codegen/CodegenContext.java @@ -140,6 +140,7 @@ public class CodegenContext { String pkg; LinkedHashSet imports = new LinkedHashSet<>(); String className; + private boolean samePackageAccess; String classModifiers = "public final"; String[] superClasses; String[] interfaces; @@ -355,6 +356,17 @@ public String getPackage() { return pkg; } + public void setSamePackageAccess(boolean samePackageAccess) { + if (this.samePackageAccess != samePackageAccess) { + sourcePublicAccessibleCache.clear(); + } + this.samePackageAccess = samePackageAccess; + } + + public boolean hasSamePackageAccess() { + return samePackageAccess; + } + public Set getValNames() { return valNames; } @@ -729,7 +741,12 @@ public String optimizeMethodCode(String code) { /** Returns true if class is public accessible from source. */ public boolean sourcePublicAccessible(Class clz) { - return sourcePublicAccessibleCache.computeIfAbsent(clz, CodeGenerator::sourcePublicAccessible); + return sourcePublicAccessibleCache.computeIfAbsent( + clz, + c -> + samePackageAccess + ? CodeGenerator.sourceAccessibleFrom(c, pkg) + : CodeGenerator.sourcePublicAccessible(c)); } /** Returns true if class is package level accessible from source. */ diff --git a/java/fory-core/src/main/java/org/apache/fory/codegen/CompileUnit.java b/java/fory-core/src/main/java/org/apache/fory/codegen/CompileUnit.java index f442c71c52..d90b6fdae4 100644 --- a/java/fory-core/src/main/java/org/apache/fory/codegen/CompileUnit.java +++ b/java/fory-core/src/main/java/org/apache/fory/codegen/CompileUnit.java @@ -31,19 +31,33 @@ public class CompileUnit { String pkg; String mainClassName; + // Non-null only when the compiled class must be defined as a JDK25+ hidden nestmate of this + // neighbor. Ordinary generated classes still use the CodeGenerator classloader path. + private final Class neighborClass; private String code; private Supplier genCodeFunc; public CompileUnit(String pkg, String mainClassName, String code) { + this(pkg, mainClassName, code, null); + } + + public CompileUnit(String pkg, String mainClassName, String code, Class neighborClass) { this.pkg = pkg; this.mainClassName = mainClassName; this.code = code; + this.neighborClass = neighborClass; } public CompileUnit(String pkg, String mainClassName, Supplier genCodeFunc) { + this(pkg, mainClassName, genCodeFunc, null); + } + + public CompileUnit( + String pkg, String mainClassName, Supplier genCodeFunc, Class neighborClass) { this.pkg = pkg; this.mainClassName = mainClassName; this.genCodeFunc = genCodeFunc; + this.neighborClass = neighborClass; } public String getCode() { @@ -65,6 +79,10 @@ public String getQualifiedClassName() { } } + public Class getNeighborClass() { + return neighborClass; + } + @Override public String toString() { return "CompileUnit{" + "pkg='" + pkg + '\'' + ", mainClassName='" + mainClassName + '\'' + '}'; diff --git a/java/fory-core/src/main/java/org/apache/fory/codegen/ExpressionOptimizer.java b/java/fory-core/src/main/java/org/apache/fory/codegen/ExpressionOptimizer.java index a9c3568e6f..68eae904bd 100644 --- a/java/fory-core/src/main/java/org/apache/fory/codegen/ExpressionOptimizer.java +++ b/java/fory-core/src/main/java/org/apache/fory/codegen/ExpressionOptimizer.java @@ -68,13 +68,12 @@ public static Expression invokeGenerated( Expression groupExpressions, String methodPrefix, boolean inlineInvoke) { + // Janino lowers private instance helpers to static bridge methods with the original binary + // class as receiver. Hidden classes have a different runtime identity, so JDK25 hidden + // generated serializers must keep split helpers non-private. + String modifier = ctx.hasSamePackageAccess() ? "final" : "private"; return invokeGenerated( - ctx, - new LinkedHashSet<>(cutPoint), - groupExpressions, - "private", - methodPrefix, - inlineInvoke); + ctx, new LinkedHashSet<>(cutPoint), groupExpressions, modifier, methodPrefix, inlineInvoke); } /** @@ -133,6 +132,7 @@ public static Expression invokeGenerated( // instance field name. CodegenContext codegenContext = new CodegenContext(ctx.getPackage(), ctx.getValNames(), ctx.getImports()); + codegenContext.setSamePackageAccess(ctx.hasSamePackageAccess()); for (Reference reference : cutExprMap.values()) { Preconditions.checkArgument(codegenContext.containName(reference.name())); } diff --git a/java/fory-core/src/main/java/org/apache/fory/platform/internal/DefineClass.java b/java/fory-core/src/main/java/org/apache/fory/platform/internal/DefineClass.java index d465038815..3b57105a41 100644 --- a/java/fory-core/src/main/java/org/apache/fory/platform/internal/DefineClass.java +++ b/java/fory-core/src/main/java/org/apache/fory/platform/internal/DefineClass.java @@ -145,7 +145,11 @@ private static RuntimeException hiddenClassFailure(Class neighbor, Throwable throw (Error) cause; } return new IllegalStateException( - "Cannot define hidden nestmate for " + neighbor.getName() + ".", cause); + "Cannot define hidden nestmate for " + + neighbor.getName() + + ". JDK25+ generated serializers require java.base/java.lang.invoke to be open to " + + "org.apache.fory.core.", + cause); } private static final class HiddenClassDefiner { diff --git a/java/fory-core/src/test/java/org/apache/fory/builder/CodecUtilsTest.java b/java/fory-core/src/test/java/org/apache/fory/builder/CodecUtilsTest.java index f9af95de99..b6b943f884 100644 --- a/java/fory-core/src/test/java/org/apache/fory/builder/CodecUtilsTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/builder/CodecUtilsTest.java @@ -20,11 +20,14 @@ package org.apache.fory.builder; import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertSame; +import static org.testng.Assert.assertTrue; import org.apache.fory.Fory; import org.apache.fory.ForyTestBase; import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.memory.MemoryUtils; +import org.apache.fory.platform.JdkVersion; import org.apache.fory.resolver.TypeResolver; import org.apache.fory.test.bean.BeanA; import org.testng.annotations.Test; @@ -41,6 +44,7 @@ public void loadOrGenObjectCodecClass() throws Exception { .requireClassRegistration(false) .build(); Class seqCodecClass = fory.getTypeResolver().getSerializerClass(BeanA.class); + assertGeneratedClassShape(seqCodecClass, BeanA.class); Generated.GeneratedSerializer serializer = seqCodecClass .asSubclass(Generated.GeneratedSerializer.class) @@ -53,4 +57,15 @@ public void loadOrGenObjectCodecClass() throws Exception { Object obj = ForyTestBase.readSerializer(fory, serializer, MemoryUtils.wrap(bytes)); assertEquals(obj, beanA); } + + private static void assertGeneratedClassShape(Class serializerClass, Class beanClass) + throws Exception { + if (JdkVersion.MAJOR_VERSION >= 25) { + assertTrue((Boolean) Class.class.getMethod("isHidden").invoke(serializerClass)); + assertSame(Class.class.getMethod("getNestHost").invoke(serializerClass), beanClass); + } else { + assertSame( + serializerClass.getClassLoader().loadClass(serializerClass.getName()), serializerClass); + } + } } From 8e4ac84051436bcc747425271b4ddefaaa976953 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Tue, 2 Jun 2026 20:18:06 +0800 Subject: [PATCH 58/69] fix(java): verify JDK25 module-path field access --- .agents/languages/java.md | 3 + ci/run_ci.sh | 1 + .../JpmsFieldAccessorTest.java | 39 +++++ java/fory-core/pom.xml | 39 +++++ .../org/apache/fory/memory/MemoryBuffer.java | 133 ++++++++---------- .../Jdk25GeneratedFieldAccessorTest.java | 92 ------------ 6 files changed, 139 insertions(+), 168 deletions(-) delete mode 100644 java/fory-latest-jdk-tests/src/test/java/org/apache/fory/integration_tests/Jdk25GeneratedFieldAccessorTest.java diff --git a/.agents/languages/java.md b/.agents/languages/java.md index e5c589a1e9..74157b2264 100644 --- a/.agents/languages/java.md +++ b/.agents/languages/java.md @@ -177,6 +177,9 @@ Load this file when changing anything under `java/` or when Java drives a cross- `initCause` or `addSuppressed` on a constructor-bypassed `Throwable` whose sentinels are still absent. - For JDK25+ CI, do not run core runtime tests from raw Maven reactor `target/classes`. Those classes bypass `META-INF/versions/25` and exercise the JDK8-24 root implementation. Build/install the multi-release artifact first, then verify the zero-Unsafe path through the JPMS module-path suite where `org.apache.fory.core` is the real access target. +- After shading Janino into `fory-core`, refresh the JPMS module descriptor package table from the + final jar before install. Otherwise named-module codegen cannot load concealed shaded Janino + packages even though the classes are present in the jar. - Do not make GraalVM native-image JDK25+ pass by opening `java.lang.invoke` to `ALL-UNNAMED`. Keep zero-Unsafe verification on JPMS JVM tests unless the native-image path itself runs Fory as a named module and the produced binary passes. ## Key Modules diff --git a/ci/run_ci.sh b/ci/run_ci.sh index 9edee67013..948e794252 100755 --- a/ci/run_ci.sh +++ b/ci/run_ci.sh @@ -61,6 +61,7 @@ install_pyfory() { } JDKS=( +"zulu26.30.11-ca-crac-jdk26.0.1-linux_x64" "zulu25.30.17-ca-jdk25.0.1-linux_x64" "zulu21.28.85-ca-jdk21.0.0-linux_x64" "zulu17.44.17-ca-crac-jdk17.0.8-linux_x64" diff --git a/integration_tests/jpms_tests/src/test/java/org/apache/fory/integration_tests/JpmsFieldAccessorTest.java b/integration_tests/jpms_tests/src/test/java/org/apache/fory/integration_tests/JpmsFieldAccessorTest.java index 9ce67d4161..ed878b1766 100644 --- a/integration_tests/jpms_tests/src/test/java/org/apache/fory/integration_tests/JpmsFieldAccessorTest.java +++ b/integration_tests/jpms_tests/src/test/java/org/apache/fory/integration_tests/JpmsFieldAccessorTest.java @@ -26,11 +26,15 @@ import org.apache.fory.integration_tests.publicserializer.PublicSerializerValue; import org.apache.fory.integration_tests.publicserializer.PublicSerializerValueSerializer; import org.apache.fory.reflect.FieldAccessor; +import org.apache.fory.serializer.CodegenSerializer.LazyInitBeanSerializer; +import org.apache.fory.serializer.Serializer; import org.testng.Assert; import org.testng.annotations.Test; public class JpmsFieldAccessorTest { private static final int JDK_MAJOR_VERSION = Runtime.version().feature(); + private static final String INSTANCE_ACCESSOR = + "org.apache.fory.reflect.InstanceFieldAccessors$InstanceAccessor"; @Test public void testPrivateFieldAccess() throws Exception { @@ -51,6 +55,24 @@ public void testPrivateFinalFieldSerialization() { Assert.assertEquals(result.value(), 13); } + @Test + public void testCodegenFinalFieldAccess() throws Exception { + if (JDK_MAJOR_VERSION < 25) { + return; + } + Fory fory = + Fory.builder().withXlang(false).withCodegen(true).requireClassRegistration(false).build(); + PrivateFieldBean result = + (PrivateFieldBean) fory.deserialize(fory.serialize(new PrivateFieldBean(17))); + Assert.assertEquals(result.value(), 17); + + Class serializerClass = serializerClass(fory, PrivateFieldBean.class); + Assert.assertTrue((Boolean) Class.class.getMethod("isHidden").invoke(serializerClass)); + Assert.assertSame( + Class.class.getMethod("getNestHost").invoke(serializerClass), PrivateFieldBean.class); + assertAccessorField(serializerClass, "value"); + } + @Test public void testReflectionFinalWriteDenied() throws Exception { if (JDK_MAJOR_VERSION < 26) { @@ -87,4 +109,21 @@ public void testPublicSerializerInExportedPackage() { Assert.assertEquals(result.value, 11); } + private static Class serializerClass(Fory fory, Class type) { + Serializer serializer = fory.getTypeResolver().getSerializer(type); + if (serializer instanceof LazyInitBeanSerializer) { + return ((LazyInitBeanSerializer) serializer).getGeneratedSerializerClass(); + } + return serializer.getClass(); + } + + private static void assertAccessorField(Class serializerClass, String fieldName) { + for (Field field : serializerClass.getDeclaredFields()) { + if (field.getName().contains(fieldName + "_accessor_")) { + Assert.assertEquals(field.getType().getName(), INSTANCE_ACCESSOR); + return; + } + } + Assert.fail("Missing generated accessor field for " + fieldName + " in " + serializerClass); + } } diff --git a/java/fory-core/pom.xml b/java/fory-core/pom.xml index e40b680c7e..e6b7ea8380 100644 --- a/java/fory-core/pom.xml +++ b/java/fory-core/pom.xml @@ -570,5 +570,44 @@ + + refresh-module-packages + + [9,) + + + + + org.apache.maven.plugins + maven-antrun-plugin + 3.1.0 + + + refresh-module-packages + package + + run + + + + + + + + + + + + + + + + + + + diff --git a/java/fory-core/src/main/java25/org/apache/fory/memory/MemoryBuffer.java b/java/fory-core/src/main/java25/org/apache/fory/memory/MemoryBuffer.java index 66308eb987..a89b97cb6b 100644 --- a/java/fory-core/src/main/java25/org/apache/fory/memory/MemoryBuffer.java +++ b/java/fory-core/src/main/java25/org/apache/fory/memory/MemoryBuffer.java @@ -57,6 +57,12 @@ *

    Warning: The instance of this class should not be held at GraalVM build time; build-time heap * buffers do not represent the runtime heap layout. * + *

    In the Java 25 multi-release implementation, absolute random-access get/put methods are + * internal fast paths. Callers must pass legal indices and ranges; read/write entry points perform + * logical {@link MemoryBuffer} range validation before reaching these methods. The implementation + * relies on {@link VarHandle}, array, and {@link ByteBuffer} access only for JVM memory safety and + * does not repeat root Unsafe-style logical bounds checks. + * *

    Note(chaokunyang): Buffer operations are very common, and jvm inline and branch elimination is * not reliable even in c2 compiler, so we try to inline and avoid checks as we can manually. jvm * jit may stop inline for some reasons: NodeCountInliningCutoff, @@ -697,57 +703,48 @@ public byte[] getArray() { // Random Access get() and put() methods // ------------------------------------------------------------------------ - private void checkPosition(long index, long pos, long length) { - if (BoundsChecking.BOUNDS_CHECKING_ENABLED) { - if (index < 0 || pos > addressLimit - length) { - throwOOBException(); - } - } - } - + /** Copies from a caller-validated absolute buffer index into {@code dst}. */ public void get(int index, byte[] dst) { get(index, dst, 0, dst.length); } + /** + * Copies from a caller-validated absolute buffer range into {@code dst}. + * + *

    This Java 25 path intentionally does not duplicate {@link MemoryBuffer} range checks; callers + * that derive {@code index}, {@code offset}, or {@code length} from wire data must validate them + * before calling this method. + */ public void get(int index, byte[] dst, int offset, int length) { + final long pos = address + index; final byte[] heapMemory = this.heapMemory; if (heapMemory != null) { // Keep heap-to-heap bulk copies on the JDK intrinsic path. System.arraycopy(heapMemory, heapOffset + index, dst, offset, length); } else { - final long pos = address + index; - if ((index - | offset - | length - | (offset + length) - | (dst.length - (offset + length)) - | addressLimit - length - pos) - < 0) { - throwOOBException(); - } readBytesToArray(pos, dst, BYTE_ARRAY_OFFSET + offset, length); } } + /** + * Copies from a caller-validated absolute buffer range into {@code target}. + * + *

    Logical range validation belongs to the caller. This method relies on the backing Java array + * or {@link ByteBuffer} accessors only to prevent invalid JVM memory access. + */ public void get(int offset, ByteBuffer target, int numBytes) { - if ((offset | numBytes | (offset + numBytes)) < 0) { - throwOOBException(); - } - if (target.remaining() < numBytes) { - throwOOBException(); - } - if (target.isReadOnly()) { - throw new IllegalArgumentException("read only buffer"); - } final int targetPos = target.position(); if (target.isDirect()) { - final long sourceAddr = address + offset; - if (sourceAddr <= addressLimit - numBytes) { - ByteBuffer duplicate = target.duplicate(); - ByteBufferUtil.position(duplicate, targetPos); - duplicate.put(sliceAsByteBuffer(offset, numBytes)); + ByteBuffer duplicate = target.duplicate(); + ByteBufferUtil.position(duplicate, targetPos); + if (heapMemory != null) { + duplicate.put(heapMemory, heapOffset + offset, numBytes); } else { - throwOOBException(); + ByteBuffer source = nativeOffHeapBuffer.duplicate(); + int sourcePos = toIntIndex(address + offset); + ByteBufferUtil.position(source, sourcePos); + source.limit(sourcePos + numBytes); + duplicate.put(source.slice()); } } else { assert target.hasArray(); @@ -758,25 +755,26 @@ public void get(int offset, ByteBuffer target, int numBytes) { } } + /** + * Copies from {@code source} into a caller-validated absolute buffer range. + * + *

    Logical range validation belongs to the caller. This method relies on the backing Java array + * or {@link ByteBuffer} accessors only to prevent invalid JVM memory access. + */ public void put(int offset, ByteBuffer source, int numBytes) { - final int remaining = source.remaining(); - if ((offset | numBytes | (offset + numBytes) | (remaining - numBytes)) < 0) { - throwOOBException(); - } final int sourcePos = source.position(); if (source.isDirect()) { - final long targetAddr = address + offset; - if (targetAddr <= addressLimit - numBytes) { - ByteBuffer duplicate = source.duplicate(); - ByteBufferUtil.position(duplicate, sourcePos); - duplicate.limit(sourcePos + numBytes); - if (heapMemory != null) { - duplicate.get(heapMemory, heapOffset + offset, numBytes); - } else { - sliceAsByteBuffer(offset, numBytes).put(duplicate.slice()); - } + ByteBuffer duplicate = source.duplicate(); + ByteBufferUtil.position(duplicate, sourcePos); + duplicate.limit(sourcePos + numBytes); + if (heapMemory != null) { + duplicate.get(heapMemory, heapOffset + offset, numBytes); } else { - throwOOBException(); + ByteBuffer target = nativeOffHeapBuffer.duplicate(); + int targetPos = toIntIndex(address + offset); + ByteBufferUtil.position(target, targetPos); + target.limit(targetPos + numBytes); + target.slice().put(duplicate.slice()); } } else { assert source.hasArray(); @@ -787,34 +785,31 @@ public void put(int offset, ByteBuffer source, int numBytes) { } } + /** Copies {@code src} into a caller-validated absolute buffer index. */ public void put(int index, byte[] src) { put(index, src, 0, src.length); } + /** + * Copies {@code src} into a caller-validated absolute buffer range. + * + *

    This Java 25 path intentionally does not duplicate {@link MemoryBuffer} range checks; callers + * that derive {@code index}, {@code offset}, or {@code length} from wire data must validate them + * before calling this method. + */ public void put(int index, byte[] src, int offset, int length) { + final long pos = address + index; final byte[] heapMemory = this.heapMemory; if (heapMemory != null) { // Keep heap-to-heap bulk copies on the JDK intrinsic path. System.arraycopy(src, offset, heapMemory, heapOffset + index, length); } else { - final long pos = address + index; - // check the byte array offset and length - if ((index - | offset - | length - | (offset + length) - | (src.length - (offset + length)) - | addressLimit - length - pos) - < 0) { - throwOOBException(); - } writeBytesFromArray(pos, src, BYTE_ARRAY_OFFSET + offset, length); } } public byte getByte(int index) { final long pos = address + index; - checkPosition(index, pos, 1); return loadByte(pos); } @@ -826,13 +821,11 @@ public byte _unsafeGetByte(int index) { public void putByte(int index, int b) { final long pos = address + index; - checkPosition(index, pos, 1); storeByte(pos, (byte) b); } public void putByte(int index, byte b) { final long pos = address + index; - checkPosition(index, pos, 1); storeByte(pos, b); } @@ -844,7 +837,6 @@ public void _unsafePutByte(int index, byte b) { public boolean getBoolean(int index) { final long pos = address + index; - checkPosition(index, pos, 1); return loadByte(pos) != 0; } @@ -855,7 +847,8 @@ public boolean _unsafeGetBoolean(int index) { } public void putBoolean(int index, boolean value) { - storeByte(address + index, (value ? (byte) 1 : (byte) 0)); + final long pos = address + index; + storeByte(pos, (value ? (byte) 1 : (byte) 0)); } // CHECKSTYLE.OFF:MethodName @@ -866,7 +859,6 @@ public void _unsafePutBoolean(int index, boolean value) { public char getChar(int index) { final long pos = address + index; - checkPosition(index, pos, 2); char c = loadChar(pos); return LITTLE_ENDIAN ? c : Character.reverseBytes(c); } @@ -880,7 +872,6 @@ public char _unsafeGetChar(int index) { public void putChar(int index, char value) { final long pos = address + index; - checkPosition(index, pos, 2); if (!LITTLE_ENDIAN) { value = Character.reverseBytes(value); } @@ -898,14 +889,12 @@ public void _unsafePutChar(int index, char value) { public short getInt16(int index) { final long pos = address + index; - checkPosition(index, pos, 2); short v = loadShort(pos); return LITTLE_ENDIAN ? v : Short.reverseBytes(v); } public void putInt16(int index, short value) { final long pos = address + index; - checkPosition(index, pos, 2); if (!LITTLE_ENDIAN) { value = Short.reverseBytes(value); } @@ -930,14 +919,12 @@ public void _unsafePutInt16(int index, short value) { public int getInt32(int index) { final long pos = address + index; - checkPosition(index, pos, 4); int v = loadInt(pos); return LITTLE_ENDIAN ? v : Integer.reverseBytes(v); } public void putInt32(int index, int value) { final long pos = address + index; - checkPosition(index, pos, 4); if (!LITTLE_ENDIAN) { value = Integer.reverseBytes(value); } @@ -962,14 +949,12 @@ public void _unsafePutInt32(int index, int value) { public long getInt64(int index) { final long pos = address + index; - checkPosition(index, pos, 8); long v = loadLong(pos); return LITTLE_ENDIAN ? v : Long.reverseBytes(v); } public void putInt64(int index, long value) { final long pos = address + index; - checkPosition(index, pos, 8); if (!LITTLE_ENDIAN) { value = Long.reverseBytes(value); } @@ -994,7 +979,6 @@ public void _unsafePutInt64(int index, long value) { public float getFloat32(int index) { final long pos = address + index; - checkPosition(index, pos, 4); int v = loadInt(pos); if (!LITTLE_ENDIAN) { v = Integer.reverseBytes(v); @@ -1004,7 +988,6 @@ public float getFloat32(int index) { public void putFloat32(int index, float value) { final long pos = address + index; - checkPosition(index, pos, 4); int v = Float.floatToRawIntBits(value); if (!LITTLE_ENDIAN) { v = Integer.reverseBytes(v); @@ -1014,7 +997,6 @@ public void putFloat32(int index, float value) { public double getFloat64(int index) { final long pos = address + index; - checkPosition(index, pos, 8); long v = loadLong(pos); if (!LITTLE_ENDIAN) { v = Long.reverseBytes(v); @@ -1024,7 +1006,6 @@ public double getFloat64(int index) { public void putFloat64(int index, double value) { final long pos = address + index; - checkPosition(index, pos, 8); long v = Double.doubleToRawLongBits(value); if (!LITTLE_ENDIAN) { v = Long.reverseBytes(v); diff --git a/java/fory-latest-jdk-tests/src/test/java/org/apache/fory/integration_tests/Jdk25GeneratedFieldAccessorTest.java b/java/fory-latest-jdk-tests/src/test/java/org/apache/fory/integration_tests/Jdk25GeneratedFieldAccessorTest.java deleted file mode 100644 index e69223878d..0000000000 --- a/java/fory-latest-jdk-tests/src/test/java/org/apache/fory/integration_tests/Jdk25GeneratedFieldAccessorTest.java +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.fory.integration_tests; - -import java.lang.reflect.Field; -import org.apache.fory.Fory; -import org.apache.fory.serializer.CodegenSerializer.LazyInitBeanSerializer; -import org.apache.fory.serializer.Serializer; -import org.testng.Assert; -import org.testng.annotations.Test; - -public class Jdk25GeneratedFieldAccessorTest { - private static final String INSTANCE_ACCESSOR = - "org.apache.fory.reflect.InstanceFieldAccessors$InstanceAccessor"; - - @Test - public void testConcreteAccessorField() { - Fory fory = - Fory.builder() - .withXlang(false) - .withCodegen(true) - .withRefTracking(false) - .requireClassRegistration(false) - .build(); - PrivateFinalBean value = new PrivateFinalBean(7, "name"); - - PrivateFinalBean copy = (PrivateFinalBean) fory.deserialize(fory.serialize(value)); - - Assert.assertEquals(copy.number(), 7); - Assert.assertEquals(copy.name(), "name"); - Class serializerClass = serializerClass(fory, PrivateFinalBean.class); - assertAccessorField(serializerClass, "number"); - assertAccessorField(serializerClass, "name"); - } - - private static Class serializerClass(Fory fory, Class type) { - Serializer serializer = fory.getTypeResolver().getSerializer(type); - if (serializer instanceof LazyInitBeanSerializer) { - return ((LazyInitBeanSerializer) serializer).getGeneratedSerializerClass(); - } - return serializer.getClass(); - } - - private static void assertAccessorField(Class serializerClass, String fieldName) { - for (Field field : serializerClass.getDeclaredFields()) { - if (field.getName().contains(fieldName + "_accessor_")) { - Assert.assertEquals(field.getType().getName(), INSTANCE_ACCESSOR); - return; - } - } - Assert.fail("Missing generated accessor field for " + fieldName + " in " + serializerClass); - } - - public static final class PrivateFinalBean { - private final int number; - private final String name; - - public PrivateFinalBean() { - this(0, null); - } - - public PrivateFinalBean(int number, String name) { - this.number = number; - this.name = name; - } - - public int number() { - return number; - } - - public String name() { - return name; - } - } -} From 16688edbe314ea14fe96540b41bd7eae71c5f913 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Tue, 2 Jun 2026 22:02:19 +0800 Subject: [PATCH 59/69] remove ForyConstructor --- .agents/languages/java.md | 35 +- .agents/languages/kotlin.md | 7 +- .github/workflows/ci.yml | 2 +- ci/run_ci.sh | 2 +- docs/guide/java/native-serialization.md | 57 +- docs/guide/kotlin/configuration.md | 25 +- docs/guide/kotlin/schema-metadata.md | 4 - .../kotlin/static-generated-serializers.md | 11 +- integration_tests/graalvm_tests/pom.xml | 19 + .../fory/graalvm/FeatureTestExample.java | 3 - .../java/org/apache/fory/graalvm/Foo.java | 2 - .../fory/graalvm/ObjectStreamExample.java | 2 - .../constructor/PrivateConstructorBean.java | 3 - .../processing/ForyStructProcessor.java | 3 +- .../StaticSerializerSourceWriter.java | 245 ------ .../processing/ForyStructProcessorTest.java | 108 +-- java/fory-core/pom.xml | 7 +- .../apache/fory/AbstractThreadSafeFory.java | 28 - .../main/java/org/apache/fory/BaseFory.java | 15 - .../src/main/java/org/apache/fory/Fory.java | 16 - .../java/org/apache/fory/ThreadLocalFory.java | 15 - .../fory/annotation/ForyConstructor.java | 44 - .../fory/builder/CompatibleCodecBuilder.java | 80 -- .../fory/builder/ObjectCodecBuilder.java | 411 +-------- .../org/apache/fory/context/ReadContext.java | 18 +- .../org/apache/fory/pool/ThreadPoolFory.java | 15 - .../apache/fory/reflect/ObjectCreator.java | 32 - .../apache/fory/reflect/ObjectCreators.java | 319 +------ .../apache/fory/resolver/SharedRegistry.java | 8 - .../serializer/AbstractObjectSerializer.java | 298 +------ .../CompatibleCollectionArrayReader.java | 4 +- .../fory/serializer/CompatibleSerializer.java | 160 +--- .../fory/serializer/ObjectSerializer.java | 89 -- .../StaticGeneratedStructSerializer.java | 41 - .../reflect/ConstructorBypassAllocator.java | 4 +- .../fory/serializer/PlatformStringUtils.java | 7 +- .../fory-core/native-image.properties | 1 - .../test/java/org/apache/fory/ForyTest.java | 6 - .../org/apache/fory/ThreadSafeForyTest.java | 102 --- .../pkgprivate/PackagePrivateMapKeyTest.java | 3 - .../fory/reflect/ObjectCreatorsTest.java | 15 - .../fory/serializer/ArraySerializersTest.java | 4 - .../fory/serializer/ObjectSerializerTest.java | 787 ------------------ .../ObjectStreamSerializerTest.java | 15 +- .../collection/CollectionSerializersTest.java | 4 - .../collection/MapSerializersTest.java | 8 - java/fory-format/pom.xml | 19 +- .../ImmutableCollectionSerializersTest.java | 2 - .../GuavaCollectionSerializersTest.java | 2 - .../org/apache/fory/test/FastJsonTest.java | 2 - .../kotlin/ksp/ForyKotlinSymbolProcessor.kt | 131 +-- .../ksp/KotlinSerializerSourceWriter.kt | 134 +-- .../kotlin/ksp/ProcessorValidationTest.kt | 33 +- .../fory/kotlin/xlang/KotlinXlangPeer.kt | 86 -- .../kotlin/KotlinDefaultValueSupport.kt | 15 +- 55 files changed, 195 insertions(+), 3313 deletions(-) delete mode 100644 java/fory-core/src/main/java/org/apache/fory/annotation/ForyConstructor.java diff --git a/.agents/languages/java.md b/.agents/languages/java.md index 74157b2264..50f236ff16 100644 --- a/.agents/languages/java.md +++ b/.agents/languages/java.md @@ -61,29 +61,26 @@ Load this file when changing anything under `java/` or when Java drives a cross- - For JDK25+ zero-Unsafe work, preserve serializer-family selection by type and configuration. Do not switch a type from `ObjectStreamSerializer` or another Fory serializer family to `JavaSerializer`, a JDK stream fallback, or any broad `java.* Serializable` fallback by JDK version or no-arg-constructor shape. - JDK25+ zero-Unsafe runtime support is a JPMS named-module design. Fory core must run with `--add-opens=java.base/java.lang.invoke=org.apache.fory.core` so it can obtain the trusted lookup; missing this open is an invalid access configuration, not a reason to open per-package JDK internals or switch serializer/object-creation families. Do not use `ALL-UNNAMED` as the zero-Unsafe verification target. - Do not probe JDK25+ trusted-lookup availability and turn `_JDKAccess` field-access booleans false when the required `java.base/java.lang.invoke` open is missing. Keep those access flags true on JDK25+ and let the owning trusted-lookup path raise the configuration error. -- Keep JDK25+ unsafe-removal implementation invariants in agent/design docs and tests, not user guides. User guides should document user actions such as `--sun-misc-unsafe-memory-access=deny`, `java.base/java.lang.invoke` opens, package opens for user named modules, and public APIs such as `@ForyConstructor` and `BaseFory.registerConstructor(...)`; do not expose internal serializer names, owner-model rationale, or avoided fallback strategies there. +- Keep JDK25+ unsafe-removal implementation invariants in agent/design docs and tests, not user guides. User guides should document user actions such as `--sun-misc-unsafe-memory-access=deny`, `java.base/java.lang.invoke` opens, and package opens for user named modules; do not expose internal serializer names, owner-model rationale, or avoided fallback strategies there. - JDK25+ zero-Unsafe final-field writes must use a true target-class trusted lookup from the original `IMPL_LOOKUP`, not `IMPL_LOOKUP.in(type)`. JDK26+ normal Fory final-field restoration must pass with `--illegal-final-field-mutation=deny` and must not require `--enable-final-field-mutation`. -- For JDK25+ object creation, do not use `sun.reflect.ReflectionFactory` or `jdk.internal.reflect.ReflectionFactory`. The shared `ObjectCreators` facade should route ObjectStream-compatible construction through `ParentNoArgCtrObjectCreator`; the replaceable constructor-bypass allocator owns the JDK8-24 Unsafe allocation path and the JDK25+ `ObjectStreamClass.newInstance` path. Serializable classes without a no-arg constructor may use that trusted-lookup owner; non-Serializable classes without a no-arg constructor require `@ForyConstructor`, `BaseFory.registerConstructor(...)`, record construction, or a custom serializer. +- For JDK25+ object creation, do not use `sun.reflect.ReflectionFactory` or `jdk.internal.reflect.ReflectionFactory`. The shared `ObjectCreators` facade should route ObjectStream-compatible construction through `ParentNoArgCtrObjectCreator`; the replaceable constructor-bypass allocator owns the JDK8-24 Unsafe allocation path and the JDK25+ `ObjectStreamClass.newInstance` path. Classes without a no-arg constructor may use that trusted-lookup owner; unsupported classes require a record canonical constructor path or a custom serializer. - In JDK25+ constructor-bypass allocation, the allocator is per type: pass the target class to the allocator constructor, cache `ObjectStreamClass.lookupAny(...)` in that instance, and expose an instance `allocate()` method. Let `ObjectStreamClass.newInstance` validate unsupported classes. Do not add redundant `Serializable` prechecks or per-call descriptor lookups, and keep exception/message construction in cold helper methods. - Keep the Java25 `_Lookup` overlay unless a future refactor can merge it without exposing Unsafe to the JDK25 class graph. Root `_Lookup` uses Unsafe for the JDK8-24 trusted-lookup fast path, while Java25 `_Lookup` uses the required `java.lang.invoke` open. `DefineClass` is root-owned; when Java25+ generated serializers need hidden nestmate class definition, it must use cached method handles and reflective `Lookup.ClassOption.NESTMATE` loading so Java 8 through Java 14 can still load the root class safely. - Treat `ByteArrayOutputStream` and `ByteArrayInputStream` as ordinary streams on every JDK. Do not restore private-buffer wrapping for JDK8-24 performance, because that reintroduces `java.base/java.io` private-field ownership and module-open requirements. - `ObjectStreamSerializer` must use its stream-specific object creator path and must not use - `TypeResolver.getObjectCreator`. `@ForyConstructor` and registered constructor mappings require - field values up front, while ObjectStream reconstruction creates the object before stream fields - are read. This path must not invoke Serializable class constructors, including no-arg + `TypeResolver.getObjectCreator`. ObjectStream reconstruction creates the object before stream + fields are read. This path must not invoke Serializable class constructors, including no-arg constructors; the JDK8-24 ReflectionFactory template constructor must be the first non-Serializable superclass no-arg constructor, and private no-arg constructors must be rejected instead of made accessible. Package-private no-arg constructors are valid only for same-package Serializable subclasses. - Runtime object creator caches are `SharedRegistry` state backed by `ConcurrentHashMap`. - Registered constructors should store the resulting `ObjectCreator` directly there, not through a - soft cache or a separate registry class. Keep ObjectStream-specific creators separate from normal - constructor-bound object creators. + Keep ObjectStream-specific creators separate from normal object creators. - Generated Fory object serializers must initialize object-creator fields through - `TypeResolver.getObjectCreator(Class)`, so generated code respects records, `@ForyConstructor`, - and `BaseFory.registerConstructor(...)`. Do not emit generated calls to + `TypeResolver.getObjectCreator(Class)`, so generated code respects runtime-scoped object creators. + Do not emit generated calls to `ObjectCreators.getObjectCreator(TypeResolver, Class)` or bypass the runtime-scoped owner; format builders without a Fory runtime context may use the base `ObjectCreators.getObjectCreator(Class)` construction default. @@ -159,16 +156,14 @@ Load this file when changing anything under `java/` or when Java drives a cross- - JDK25+ collection serializers must fail unsupported `Collections.newSetFromMap` backing maps before writing or copying. Do not rewrite them to `HashMap`, because that changes equality semantics and can drop entries. -- Do not enable Java or Kotlin parameter-name metadata (`-parameters`, `maven.compiler.parameters`, or Kotlin `javaParameters`) to make constructor binding work. Constructor binding must be driven by explicit field mappings from `@ForyConstructor`, `BaseFory.registerConstructor(...)`, or generated metadata. -- Constructor-bound serializers must cache constructor field metadata during serializer initialization. Do not call defensive-copy metadata getters such as `getConstructorFieldTypes()` from per-object read paths. -- Explicit constructor binding is for user and third-party classes, not Java platform types. Do not use - `java.*` internals such as `String` fields as constructor-binding test data; JDK built-in classes - are owned by their serializers. -- Runtime serializers and generated serializers must use the same constructor-copy lifecycle: - install a pending marker before copying constructor-bound fields, run field serializers normally, - reject copied constructor arguments that still retain the marker, then construct and reference the - real copy. Do not implement a separate raw-field pre-scan or leave a generated path without the - marker guard. +- Do not add or restore constructor-binding APIs such as `@ForyConstructor` or + `BaseFory.registerConstructor(...)`. Java parameter names, `-parameters`, and + `@ConstructorProperties` are not a Fory object-creation contract. Runtime serializers for + ordinary classes must create an empty instance through `TypeResolver.getObjectCreator(Class)` and + set fields; records and source-generated Kotlin serializers are the constructor-owned paths. +- Source-generated constructor serializers must own their constructor metadata at generation time + and call constructors directly. They must not depend on runtime `ObjectCreator` constructor-field + metadata or varargs constructor calls. - Generated JVM copy code may direct-copy immutable scalar values, but Java `Collection`/`Map` subclasses must be copied through `CopyContext.copyObject(...)` so collection/map serializers own concrete type, comparator, wrapper, and reference behavior. diff --git a/.agents/languages/kotlin.md b/.agents/languages/kotlin.md index 7e45aab2f7..2df307a96e 100644 --- a/.agents/languages/kotlin.md +++ b/.agents/languages/kotlin.md @@ -6,9 +6,10 @@ Load this file when changing `kotlin/`. - Run Kotlin Maven commands from within `kotlin/`. - Kotlin serializers build on the Java implementation. If Java changed and the updated Java artifacts are not installed yet, run `cd ../java && mvn -T16 install -DskipTests` first. -- KSP `@ForyStruct` serializers that use a primary constructor must require explicit - `@ForyConstructor` field mappings. Do not bind constructor fields from Kotlin parameter names or - `javaParameters`; mutable no-argument structs should use `var` properties with `@ForyField`. +- KSP `@ForyStruct` serializers that use a primary constructor map constructor parameters to + same-named source properties at generation time and call the constructor directly. Do not restore + `@ForyConstructor`, runtime constructor registration, or Kotlin `javaParameters` dependencies; + mutable no-argument structs should use `var` properties with `@ForyField`. - Preserve serializer-family selection for Kotlin standard-library types already registered by Fory. Do not auto-install a new serializer for an existing type-registered Kotlin class unless the wire format matches the previous serializer family and old-payload/new-runtime compatibility is diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d7baf5ab5b..d9c794f51c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -516,7 +516,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - java-version: ["17", "21"] + java-version: ["17", "21", "25"] steps: - uses: actions/checkout@v5 - uses: graalvm/setup-graalvm@f744c72a42b1995d7b0cbc314bde4bace7ac1fe1 # 1.5.0 diff --git a/ci/run_ci.sh b/ci/run_ci.sh index 948e794252..5798e9972e 100755 --- a/ci/run_ci.sh +++ b/ci/run_ci.sh @@ -87,7 +87,7 @@ graalvm_test() { java_major=$(echo "$java_version" | cut -d. -f1) fi if [[ "$java_major" -ge 25 ]]; then - export JDK_JAVA_OPTIONS="$(jdk25_javac_options)" + export JDK_JAVA_OPTIONS="$(jdk25_plus_options "$java_major" "ALL-UNNAMED") $(jdk25_javac_options)" else unset JDK_JAVA_OPTIONS fi diff --git a/docs/guide/java/native-serialization.md b/docs/guide/java/native-serialization.md index 23a24fca5d..3129a8a9ec 100644 --- a/docs/guide/java/native-serialization.md +++ b/docs/guide/java/native-serialization.md @@ -165,57 +165,20 @@ path is too expensive. ## Final Fields And Constructors Records are deserialized through their canonical constructor. Ordinary classes use Fory's normal -object-creation path and field setting unless you provide an explicit constructor mapping. - -Use `@ForyConstructor` when a constructor should receive serialized field values: - -```java -import org.apache.fory.annotation.ForyConstructor; - -public final class User { - private final String name; - private final int age; - - @ForyConstructor({"name", "age"}) - public User(String name, int age) { - this.name = name; - this.age = age; - } -} -``` - -For third-party classes that cannot be annotated, register the constructor during runtime setup -before requesting serializers or starting serialization, deserialization, or copy operations: - -```java -import java.lang.reflect.Constructor; - -Constructor constructor = User.class.getDeclaredConstructor(String.class, int.class); -fory.registerConstructor(User.class, constructor, "name", "age"); -``` - -The field names are the binding contract. For ordinary classes, Fory does not infer constructor -bindings from Java parameter names, `-parameters`, or `@ConstructorProperties`. -Explicit constructor mappings must bind at least one field. Leave ordinary no-argument constructors -unannotated and unregistered. Do not register constructors for Java platform classes; Fory owns -their built-in serializers. - -When no explicit constructor mapping is provided, normal classes with final fields use Fory's normal -object creation and field setting. On JDK25+ with Unsafe memory access denied, Fory reports an error -if the class cannot be created by supported Java mechanisms. Use `@ForyConstructor`, -`registerConstructor(...)`, a record canonical constructor, or a custom serializer for those classes. -Use the `java.base/java.lang.invoke` open shown in troubleshooting for supported JDK25+ access paths. -Fory does not require `--enable-final-field-mutation` for ordinary final-field restoration on +object-creation path and field setting, including final fields when the runtime supports it. Fory +does not expose a constructor-mapping annotation or a constructor-registration API for ordinary +classes, and Java parameter-name metadata such as `-parameters` or `@ConstructorProperties` is not a +Fory object-creation contract. + +If an ordinary class cannot be created by supported Java mechanisms, use an accessible no-argument +constructor, model it as a record when canonical-constructor semantics are appropriate, or register a +custom serializer. On JDK25+ with Unsafe memory access denied, use the +`java.base/java.lang.invoke` open shown in troubleshooting for supported final-field and JDK access +paths. Fory does not require `--enable-final-field-mutation` for ordinary final-field restoration on JDK26+. See [Troubleshooting](troubleshooting.md#jdk25-zero-unsafe-mode-and-module-opens) for the required JVM flags. -Constructor-bound objects cannot receive a constructor argument that retains a direct or nested -back-reference to the same object under construction. Field serializers may transform or omit data -during copy; Fory rejects only back-references that remain in the copied constructor arguments. Model -those cycles through non-constructor fields or use a custom serializer that removes the constructor -argument cycle. - ## JDK Serialization Hooks Java native mode supports the JDK serialization hooks that are part of many existing Java object diff --git a/docs/guide/kotlin/configuration.md b/docs/guide/kotlin/configuration.md index b4b2880551..23004d35b6 100644 --- a/docs/guide/kotlin/configuration.md +++ b/docs/guide/kotlin/configuration.md @@ -99,25 +99,12 @@ All configuration options from Fory Java are available. See [Java Configuration] ## JDK25+ Zero-Unsafe Mode -On JDK25+ with Unsafe memory access denied, Kotlin classes with final constructor properties need -an explicit constructor mapping when Fory must call the primary constructor. Annotate the -constructor with `@ForyConstructor`, register the constructor with `registerConstructor(...)`, use a -generated serializer that carries explicit constructor metadata, or provide a custom serializer. -Register constructors during setup before requesting serializers or starting serialization, -deserialization, or copy operations. - -```kotlin -import org.apache.fory.annotation.ForyConstructor - -class User @ForyConstructor("name", "age") constructor( - val name: String, - val age: Int, -) -``` - -KSP-generated `@ForyStruct` serializers that call a primary constructor require the same explicit -`@ForyConstructor` mapping. Mutable no-argument `@ForyStruct` classes can instead expose serialized -`var` properties with `@ForyField`. +On JDK25+ with Unsafe memory access denied, Kotlin classes follow the same final-field rules as Java +native serialization. Ordinary runtime serializers use Fory's supported object-creation path and +field setting; KSP-generated `@ForyStruct` serializers call source-visible primary constructors +directly. Fory does not expose constructor-mapping APIs for normal Kotlin classes. If a class cannot +be created by supported Java mechanisms, use an accessible no-argument constructor, a generated +`@ForyStruct` serializer, or a custom serializer. The JVM also needs the module opens listed in [Java Troubleshooting](../java/troubleshooting.md#jdk25-zero-unsafe-mode-and-module-opens). diff --git a/docs/guide/kotlin/schema-metadata.md b/docs/guide/kotlin/schema-metadata.md index b27720d9b2..7b4ed24a97 100644 --- a/docs/guide/kotlin/schema-metadata.md +++ b/docs/guide/kotlin/schema-metadata.md @@ -29,7 +29,6 @@ Annotate Kotlin schema classes with `@ForyStruct` and constructor properties wit `@ForyField(id = N)`: ```kotlin -import org.apache.fory.annotation.ForyConstructor import org.apache.fory.annotation.ForyField import org.apache.fory.annotation.ForyStruct import org.apache.fory.kotlin.Fixed @@ -37,7 +36,6 @@ import org.apache.fory.kotlin.VarInt @ForyStruct data class User -@ForyConstructor("id", "score", "tags") constructor( @ForyField(id = 1) val id: @Fixed UInt, @@ -62,7 +60,6 @@ and maps: ```kotlin @ForyStruct data class NullabilityExample -@ForyConstructor("names", "optionalNames", "nullableList") constructor( @ForyField(id = 1) val names: List, @@ -87,7 +84,6 @@ import org.apache.fory.annotation.Ref @ForyStruct data class Node -@ForyConstructor("children", "parent") constructor( @ForyField(id = 1) val children: List<@Ref Node>, diff --git a/docs/guide/kotlin/static-generated-serializers.md b/docs/guide/kotlin/static-generated-serializers.md index 3bf3d2c713..0282c84781 100644 --- a/docs/guide/kotlin/static-generated-serializers.md +++ b/docs/guide/kotlin/static-generated-serializers.md @@ -54,7 +54,6 @@ Reuse the Java Fory annotations for schema concepts. Use Kotlin type-use annotations only when you need to override integer encoding. ```kotlin -import org.apache.fory.annotation.ForyConstructor import org.apache.fory.annotation.ForyField import org.apache.fory.annotation.ForyStruct import org.apache.fory.kotlin.Fixed @@ -62,7 +61,6 @@ import org.apache.fory.kotlin.VarInt @ForyStruct data class User -@ForyConstructor("id", "score", "tags") constructor( @ForyField(id = 1) val id: @Fixed UInt, @@ -84,10 +82,10 @@ them. The processor generates serializers for public or internal, concrete, non-generic classes in named packages. A supported class must have a primary -constructor whose serialized parameters are `val` or `var` properties and whose -constructor parameters are explicitly mapped with `@ForyConstructor`. `data -class` is the common case, but it is not required. Mutable no-argument structs -can instead expose serialized `var` properties with `@ForyField`. +constructor whose serialized parameters are `val` or `var` properties with the +same names as the constructor parameters. `data class` is the common case, but it +is not required. Mutable no-argument structs can instead expose serialized `var` +properties with `@ForyField`. Internal Kotlin struct classes are supported when KSP runs in the same Kotlin module that owns the struct. The generated Kotlin serializer is also internal, @@ -121,7 +119,6 @@ inside collections and maps. ```kotlin @ForyStruct data class NullabilityExample -@ForyConstructor("a", "b", "c", "d") constructor( @ForyField(id = 1) val a: List, diff --git a/integration_tests/graalvm_tests/pom.xml b/integration_tests/graalvm_tests/pom.xml index dc211053db..a070e22b97 100644 --- a/integration_tests/graalvm_tests/pom.xml +++ b/integration_tests/graalvm_tests/pom.xml @@ -140,6 +140,25 @@ + + native-jdk25 + + [25,) + + + + + org.graalvm.buildtools + native-maven-plugin + + + -J--add-opens=java.base/java.lang.invoke=ALL-UNNAMED + + + + + + native diff --git a/integration_tests/graalvm_tests/src/main/java/org/apache/fory/graalvm/FeatureTestExample.java b/integration_tests/graalvm_tests/src/main/java/org/apache/fory/graalvm/FeatureTestExample.java index 43a202d170..1e0f17d7bc 100644 --- a/integration_tests/graalvm_tests/src/main/java/org/apache/fory/graalvm/FeatureTestExample.java +++ b/integration_tests/graalvm_tests/src/main/java/org/apache/fory/graalvm/FeatureTestExample.java @@ -23,7 +23,6 @@ import java.lang.reflect.Method; import java.lang.reflect.Proxy; import org.apache.fory.Fory; -import org.apache.fory.annotation.ForyConstructor; import org.apache.fory.builder.Generated; import org.apache.fory.platform.GraalvmSupport; import org.apache.fory.util.Preconditions; @@ -51,7 +50,6 @@ public interface TestInterface { public static class TestInvocationHandler implements InvocationHandler { private final String value; - @ForyConstructor("value") public TestInvocationHandler(String value) { this.value = value; } @@ -100,7 +98,6 @@ public static void main(String[] args) { // Test proxy serialization TestInterface proxy = - (TestInterface) Proxy.newProxyInstance( TestInterface.class.getClassLoader(), new Class[] {TestInterface.class}, diff --git a/integration_tests/graalvm_tests/src/main/java/org/apache/fory/graalvm/Foo.java b/integration_tests/graalvm_tests/src/main/java/org/apache/fory/graalvm/Foo.java index d4ad4a6b8b..3aa1362faf 100644 --- a/integration_tests/graalvm_tests/src/main/java/org/apache/fory/graalvm/Foo.java +++ b/integration_tests/graalvm_tests/src/main/java/org/apache/fory/graalvm/Foo.java @@ -23,7 +23,6 @@ import java.util.List; import java.util.Map; import java.util.Objects; -import org.apache.fory.annotation.ForyConstructor; public class Foo implements Serializable { int f1; @@ -31,7 +30,6 @@ public class Foo implements Serializable { List f3; Map f4; - @ForyConstructor({"f1", "f2", "f3", "f4"}) public Foo(int f1, String f2, List f3, Map f4) { this.f1 = f1; this.f2 = f2; diff --git a/integration_tests/graalvm_tests/src/main/java/org/apache/fory/graalvm/ObjectStreamExample.java b/integration_tests/graalvm_tests/src/main/java/org/apache/fory/graalvm/ObjectStreamExample.java index c7978eb8e1..51693a22d5 100644 --- a/integration_tests/graalvm_tests/src/main/java/org/apache/fory/graalvm/ObjectStreamExample.java +++ b/integration_tests/graalvm_tests/src/main/java/org/apache/fory/graalvm/ObjectStreamExample.java @@ -27,7 +27,6 @@ import java.util.TreeMap; import java.util.TreeSet; import org.apache.fory.Fory; -import org.apache.fory.annotation.ForyConstructor; import org.apache.fory.serializer.ObjectStreamSerializer; import org.apache.fory.serializer.collection.CollectionSerializers; import org.apache.fory.serializer.collection.MapSerializers; @@ -73,7 +72,6 @@ public ObjectStreamExample() { this(new int[10]); } - @ForyConstructor("ints") public ObjectStreamExample(int[] ints) { this.ints = ints; } diff --git a/integration_tests/jpms_tests/src/main/java/org/apache/fory/integration_tests/constructor/PrivateConstructorBean.java b/integration_tests/jpms_tests/src/main/java/org/apache/fory/integration_tests/constructor/PrivateConstructorBean.java index 55187043b9..04434443f8 100644 --- a/integration_tests/jpms_tests/src/main/java/org/apache/fory/integration_tests/constructor/PrivateConstructorBean.java +++ b/integration_tests/jpms_tests/src/main/java/org/apache/fory/integration_tests/constructor/PrivateConstructorBean.java @@ -19,13 +19,10 @@ package org.apache.fory.integration_tests.constructor; -import org.apache.fory.annotation.ForyConstructor; - public final class PrivateConstructorBean { private final String name; private final int age; - @ForyConstructor({"name", "age"}) private PrivateConstructorBean(String name, int age) { this.name = name; this.age = age; diff --git a/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/ForyStructProcessor.java b/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/ForyStructProcessor.java index 0867d57245..2eb7e7d496 100644 --- a/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/ForyStructProcessor.java +++ b/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/ForyStructProcessor.java @@ -61,8 +61,7 @@ @SupportedAnnotationTypes({ "org.apache.fory.annotation.ForyStruct", - "org.apache.fory.annotation.ForyDebug", - "org.apache.fory.annotation.ForyConstructor" + "org.apache.fory.annotation.ForyDebug" }) public final class ForyStructProcessor extends AbstractProcessor { private static final String ARRAY_TYPE = "org.apache.fory.annotation.ArrayType"; diff --git a/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/StaticSerializerSourceWriter.java b/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/StaticSerializerSourceWriter.java index 4fdabc911c..f439a2c140 100644 --- a/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/StaticSerializerSourceWriter.java +++ b/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/StaticSerializerSourceWriter.java @@ -88,9 +88,6 @@ private void writeClassStart() { builder.append(" private final SerializationFieldInfo[] allFields;\n"); builder.append(" private final int[] allFieldIds;\n"); builder.append(" private final SerializationFieldInfo[] fieldsById;\n"); - builder.append(" private final int[] constructorFieldIds;\n"); - builder.append(" private final long[] constructorFieldBits;\n"); - builder.append(" private final Class[] constructorFieldTypes;\n"); builder.append(" private final int classVersionHash;\n"); builder.append(" private final boolean sameSchemaCompatible;\n\n"); } @@ -143,9 +140,6 @@ private void writeConstructors() { builder.append(" this.allFields = null;\n"); builder.append(" this.allFieldIds = null;\n"); builder.append(" this.fieldsById = null;\n"); - builder.append(" this.constructorFieldIds = null;\n"); - builder.append(" this.constructorFieldBits = null;\n"); - builder.append(" this.constructorFieldTypes = null;\n"); builder.append(" this.classVersionHash = 0;\n"); builder.append(" this.sameSchemaCompatible = false;\n"); builder.append(" }\n\n"); @@ -177,12 +171,6 @@ private void writeConstructorBody(String fieldGroupsExpression, String sameSchem builder.append(" for (int i = 0; i < allFields.length; i++) {\n"); builder.append(" this.fieldsById[allFieldIds[i]] = allFields[i];\n"); builder.append(" }\n"); - builder.append( - " this.constructorFieldIds = objectCreator.hasConstructorFields() ? buildConstructorFieldIds(DESCRIPTORS) : null;\n"); - builder.append( - " this.constructorFieldBits = buildConstructorFieldBits(DESCRIPTORS.size(), constructorFieldIds);\n"); - builder.append( - " this.constructorFieldTypes = constructorFieldIds != null ? constructorFieldTypes() : null;\n"); builder.append( " this.classVersionHash = typeResolver.checkClassVersion() ? computeClassVersionHash(DESCRIPTORS) : 0;\n"); builder.append(" this.sameSchemaCompatible = ").append(sameSchemaExpression).append(";\n"); @@ -247,9 +235,6 @@ private void writeSchemaConsistentRead() { appendRecordConstructorArguments("field"); builder.append(");\n"); } else { - builder.append(" if (constructorFieldIds != null) {\n"); - builder.append(" return readSchemaConsistentConstructor(readContext);\n"); - builder.append(" }\n"); builder.append(" ").append(struct.typeName).append(" value = newBean();\n"); builder.append(" readContext.reference(value);\n"); builder.append(" readFields(readContext, value);\n"); @@ -307,7 +292,6 @@ private void writeReadGroups() { writeReadRecordGroup("", "allFields", "allFieldIds", "readFieldValue"); } else { writeReadBeanGroup("", "allFields", "allFieldIds", "readFieldValue"); - writeConstructorRead(); } } @@ -377,170 +361,6 @@ private void writeReadBeanGroup( builder.append(" }\n\n"); } - private void writeConstructorRead() { - builder - .append(" private ") - .append(struct.typeName) - .append(" readSchemaConsistentConstructor(ReadContext readContext) {\n"); - builder.append(" Object[] fieldValues = new Object[DESCRIPTORS.size()];\n"); - builder.append(" long[] bufferedFields = newFieldBits(DESCRIPTORS.size());\n"); - builder.append(" beginConstructorRef(readContext);\n"); - builder.append(" try {\n"); - builder.append(" int remaining = countConstructorFields(constructorFieldBits);\n"); - builder.append(" ").append(struct.typeName).append(" value = null;\n"); - builder.append(" if (remaining == 0) {\n"); - builder.append(" value = newConstructorObject(fieldValues);\n"); - builder.append(" referenceConstructorRef(readContext, value);\n"); - builder.append(" }\n"); - builder.append(" for (int i = 0; i < allFields.length; i++) {\n"); - builder.append(" SerializationFieldInfo fieldInfo = allFields[i];\n"); - builder.append(" int fieldId = allFieldIds[i];\n"); - builder.append(" if (hasField(constructorFieldBits, fieldId)) {\n"); - builder.append( - " fieldValues[fieldId] = ctorFieldValue(readContext, readFieldValue(readContext, fieldInfo), type);\n"); - builder.append(" remaining--;\n"); - builder.append(" if (remaining == 0) {\n"); - builder.append(" checkNoUnresolvedReadRef(readContext);\n"); - builder.append(" value = newConstructorObject(fieldValues);\n"); - builder.append(" referenceConstructorRef(readContext, value);\n"); - builder.append(" setBufferedFields(value, fieldValues, bufferedFields);\n"); - builder.append(" }\n"); - builder.append(" } else if (value == null) {\n"); - builder.append( - " fieldValues[fieldId] = bufferFieldValue(readContext, readFieldValue(readContext, fieldInfo), type);\n"); - builder.append(" markField(bufferedFields, fieldId);\n"); - builder.append(" } else {\n"); - builder.append(" readAndSetField(readContext, value, fieldInfo, fieldId);\n"); - builder.append(" }\n"); - builder.append(" }\n"); - builder.append(" if (value == null) {\n"); - builder.append(" checkNoUnresolvedReadRef(readContext);\n"); - builder.append(" value = newConstructorObject(fieldValues);\n"); - builder.append(" referenceConstructorRef(readContext, value);\n"); - builder.append(" setBufferedFields(value, fieldValues, bufferedFields);\n"); - builder.append(" }\n"); - builder.append(" return value;\n"); - builder.append(" } finally {\n"); - builder.append(" endConstructorRef(readContext);\n"); - builder.append(" }\n"); - builder.append(" }\n\n"); - - builder - .append(" private ") - .append(struct.typeName) - .append(" readCompatibleConstructor(ReadContext readContext) {\n"); - builder.append(" Object[] fieldValues = new Object[DESCRIPTORS.size()];\n"); - builder.append(" long[] bufferedFields = newFieldBits(DESCRIPTORS.size());\n"); - builder.append(" beginConstructorRef(readContext);\n"); - builder.append(" try {\n"); - builder.append(" int remaining = countConstructorFields(constructorFieldBits);\n"); - builder.append(" ").append(struct.typeName).append(" value = null;\n"); - builder.append(" if (remaining == 0) {\n"); - builder.append(" value = newConstructorObject(fieldValues);\n"); - builder.append(" referenceConstructorRef(readContext, value);\n"); - builder.append(" }\n"); - builder.append(" for (int i = 0; i < remoteFields.size(); i++) {\n"); - builder.append(" RemoteFieldInfo remoteField = remoteFields.get(i);\n"); - builder.append(" int fieldId = remoteField.matchedId;\n"); - builder.append(" if (fieldId < 0) {\n"); - builder.append(" skipField(readContext, remoteField);\n"); - builder.append(" continue;\n"); - builder.append(" }\n"); - builder.append(" SerializationFieldInfo localField = fieldsById[fieldId];\n"); - builder.append(" if (!canReadRemoteField(remoteField, localField)) {\n"); - builder.append(" skipField(readContext, remoteField);\n"); - builder.append(" continue;\n"); - builder.append(" }\n"); - builder.append( - " Object fieldValue = readCompatibleFieldValue(readContext, remoteField, localField);\n"); - builder.append(" if (hasField(constructorFieldBits, fieldId)) {\n"); - builder.append( - " fieldValues[fieldId] = ctorFieldValue(readContext, fieldValue, type);\n"); - builder.append(" remaining--;\n"); - builder.append(" if (remaining == 0) {\n"); - builder.append(" checkNoUnresolvedReadRef(readContext);\n"); - builder.append(" value = newConstructorObject(fieldValues);\n"); - builder.append(" referenceConstructorRef(readContext, value);\n"); - builder.append(" setBufferedFields(value, fieldValues, bufferedFields);\n"); - builder.append(" }\n"); - builder.append(" } else if (value == null) {\n"); - builder.append( - " fieldValues[fieldId] = bufferFieldValue(readContext, fieldValue, type);\n"); - builder.append(" markField(bufferedFields, fieldId);\n"); - builder.append(" } else {\n"); - builder.append(" setFieldById(value, localField, fieldId, fieldValue);\n"); - builder.append(" }\n"); - builder.append(" }\n"); - builder.append(" if (value == null) {\n"); - builder.append(" checkNoUnresolvedReadRef(readContext);\n"); - builder.append(" value = newConstructorObject(fieldValues);\n"); - builder.append(" referenceConstructorRef(readContext, value);\n"); - builder.append(" setBufferedFields(value, fieldValues, bufferedFields);\n"); - builder.append(" }\n"); - builder.append(" return value;\n"); - builder.append(" } finally {\n"); - builder.append(" endConstructorRef(readContext);\n"); - builder.append(" }\n"); - builder.append(" }\n\n"); - - builder - .append(" private ") - .append(struct.typeName) - .append(" newConstructorObject(Object[] fieldValues) {\n"); - builder - .append(" return (") - .append(struct.typeName) - .append( - ") objectCreator.newInstanceWithArguments(constructorArgs(fieldValues, constructorFieldIds, constructorFieldTypes));\n"); - builder.append(" }\n\n"); - - builder - .append(" private void readAndSetField(ReadContext readContext, ") - .append(struct.typeName) - .append(" value, SerializationFieldInfo fieldInfo, int fieldId) {\n"); - builder.append(" Object fieldValue = readFieldValue(readContext, fieldInfo);\n"); - builder.append(" setFieldById(value, fieldInfo, fieldId, fieldValue);\n"); - builder.append(" }\n\n"); - - builder - .append(" private void setBufferedFields(") - .append(struct.typeName) - .append(" value, Object[] fieldValues, long[] bufferedFields) {\n"); - builder.append(" for (int fieldId = 0; fieldId < fieldsById.length; fieldId++) {\n"); - builder.append(" if (hasField(bufferedFields, fieldId)) {\n"); - builder.append( - " setFieldById(value, fieldsById[fieldId], fieldId, resolveBufferedValue(fieldValues[fieldId], value));\n"); - builder.append(" }\n"); - builder.append(" }\n"); - builder.append(" }\n\n"); - - builder - .append(" private void setFieldById(") - .append(struct.typeName) - .append(" value, SerializationFieldInfo fieldInfo, int fieldId, Object fieldValue) {\n"); - builder.append(" switch (fieldId) {\n"); - for (SourceField field : struct.fields) { - builder.append(" case ").append(field.id).append(":\n"); - builder.append(" "); - if (field.finalField) { - builder - .append("setGeneratedFieldValue(value, fieldInfo, ") - .append(field.castExpression("fieldValue")) - .append(");\n"); - } else { - builder - .append(field.writeStatement("value", field.castExpression("fieldValue"))) - .append("\n"); - } - builder.append(" return;\n"); - } - builder.append(" default:\n"); - builder.append( - " throw new IllegalStateException(\"Unknown generated field id \" + fieldId);\n"); - builder.append(" }\n"); - builder.append(" }\n\n"); - } - private boolean hasDirectWriteField() { for (SourceField field : struct.fields) { if (canEmitDirectWriteField(field)) { @@ -1019,9 +839,6 @@ private void writeCompatibleRead() { appendRecordConstructorArguments("field"); builder.append(");\n"); } else { - builder.append(" if (constructorFieldIds != null) {\n"); - builder.append(" return readCompatibleConstructor(readContext);\n"); - builder.append(" }\n"); builder.append(" ").append(struct.typeName).append(" value = newBean();\n"); builder.append(" readContext.reference(value);\n"); builder.append(" for (int i = 0; i < remoteFields.size(); i++) {\n"); @@ -1237,11 +1054,6 @@ private void writeCopy() { builder.append(" if (immutable) {\n"); builder.append(" return value;\n"); builder.append(" }\n"); - if (!struct.record) { - builder.append(" if (constructorFieldIds != null) {\n"); - builder.append(" return copyConstructorObject(copyContext, value);\n"); - builder.append(" }\n"); - } if (struct.record) { for (SourceField field : struct.fields) { builder @@ -1295,63 +1107,6 @@ private void writeCopy() { builder.append(" return copied;\n"); } builder.append(" }\n\n"); - if (!struct.record) { - writeConstructorCopy(); - } - } - - private void writeConstructorCopy() { - builder - .append(" private ") - .append(struct.typeName) - .append(" copyConstructorObject(CopyContext copyContext, ") - .append(struct.typeName) - .append(" value) {\n"); - builder.append(" Object[] fieldValues = new Object[DESCRIPTORS.size()];\n"); - builder.append(" Object pendingMarker = beginConstructorCopy(copyContext, value);\n"); - for (SourceField field : struct.fields) { - builder.append(" if (hasField(constructorFieldBits, ").append(field.id).append(")) {\n"); - builder - .append(" fieldValues[") - .append(field.id) - .append("] = copyConstructorFieldValue(copyContext, value, ") - .append(field.readExpression("value")) - .append(", fieldsById[") - .append(field.id) - .append("]);\n"); - builder.append(" }\n"); - } - builder.append( - " checkNoConstructorCopyBackrefs(fieldValues, constructorFieldIds, pendingMarker);\n"); - builder - .append(" ") - .append(struct.typeName) - .append(" copied = newConstructorObject(fieldValues);\n"); - builder.append(" copyContext.reference(value, copied);\n"); - for (SourceField field : struct.fields) { - builder.append(" if (!hasField(constructorFieldBits, ").append(field.id).append(")) {\n"); - String copiedExpression = - field.castExpression( - "copyFieldValue(copyContext, " - + field.readExpression("value") - + ", fieldsById[" - + field.id - + "])"); - builder.append(" "); - if (field.finalField) { - builder - .append("setGeneratedFieldValue(copied, fieldsById[") - .append(field.id) - .append("], ") - .append(copiedExpression) - .append(");\n"); - } else { - builder.append(field.writeStatement("copied", copiedExpression)).append("\n"); - } - builder.append(" }\n"); - } - builder.append(" return copied;\n"); - builder.append(" }\n\n"); } private void writeDescriptorHelpers() { diff --git a/java/fory-annotation-processor/src/test/java/org/apache/fory/annotation/processing/ForyStructProcessorTest.java b/java/fory-annotation-processor/src/test/java/org/apache/fory/annotation/processing/ForyStructProcessorTest.java index d5faa619aa..09a7e21d4e 100644 --- a/java/fory-annotation-processor/src/test/java/org/apache/fory/annotation/processing/ForyStructProcessorTest.java +++ b/java/fory-annotation-processor/src/test/java/org/apache/fory/annotation/processing/ForyStructProcessorTest.java @@ -20,7 +20,6 @@ package org.apache.fory.annotation.processing; import java.io.IOException; -import java.lang.reflect.Constructor; import java.lang.reflect.Field; import java.net.URL; import java.net.URLClassLoader; @@ -42,7 +41,6 @@ import org.apache.fory.context.MetaReadContext; import org.apache.fory.context.MetaWriteContext; import org.apache.fory.exception.DeserializationException; -import org.apache.fory.exception.ForyException; import org.apache.fory.exception.SerializationException; import org.apache.fory.meta.FieldInfo; import org.apache.fory.meta.TypeDef; @@ -118,19 +116,17 @@ public void testLegacyBooleanEvolvingAnnotationCompiles() throws Exception { } @Test - public void testStaticAnnotatedConstructor() throws Exception { + public void testStaticFinalFields() throws Exception { CompilationResult result = compile( - "test.AnnotatedConstructorStruct", + "test.FinalFieldStruct", "package test;\n" - + "import org.apache.fory.annotation.ForyConstructor;\n" + "import org.apache.fory.annotation.ForyStruct;\n" - + "@ForyStruct public class AnnotatedConstructorStruct {\n" + + "@ForyStruct public class FinalFieldStruct {\n" + " private final String name;\n" + " private final int age;\n" + " public String note;\n" - + " @ForyConstructor({\"name\", \"age\"})\n" - + " public AnnotatedConstructorStruct(String name, int age) {\n" + + " public FinalFieldStruct(String name, int age) {\n" + " this.name = name;\n" + " this.age = age;\n" + " }\n" @@ -138,14 +134,11 @@ public void testStaticAnnotatedConstructor() throws Exception { + " public int getAge() { return age; }\n" + "}\n"); Assert.assertTrue(result.success, result.diagnostics()); - String generatedSource = - result.generatedSource("test/AnnotatedConstructorStruct_ForyNativeSerializer.java"); - Assert.assertFalse(generatedSource.contains("boolean[]"), generatedSource); - Assert.assertTrue(generatedSource.contains("long[] constructorFieldBits"), generatedSource); - Assert.assertTrue( - generatedSource.contains("newFieldBits(DESCRIPTORS.size())"), generatedSource); + String generatedSource = result.generatedSource("test/FinalFieldStruct_ForyNativeSerializer.java"); + Assert.assertFalse(generatedSource.contains("constructorFieldBits"), generatedSource); + Assert.assertTrue(generatedSource.contains("setGeneratedFieldValue"), generatedSource); try (URLClassLoader loader = result.classLoader()) { - Class type = loader.loadClass("test.AnnotatedConstructorStruct"); + Class type = loader.loadClass("test.FinalFieldStruct"); Object value = type.getConstructor(String.class, int.class).newInstance("fory", 12); setField(type, value, "note", "static"); Fory fory = @@ -169,91 +162,6 @@ public void testStaticAnnotatedConstructor() throws Exception { } } - @Test - @SuppressWarnings({"unchecked", "rawtypes"}) - public void testStaticRegisteredConstructor() throws Exception { - CompilationResult result = - compile( - "test.RegisteredConstructorStruct", - "package test;\n" - + "import org.apache.fory.annotation.ForyStruct;\n" - + "@ForyStruct public class RegisteredConstructorStruct {\n" - + " public static int constructorCalls;\n" - + " private final String name;\n" - + " private final int age;\n" - + " public String note;\n" - + " public RegisteredConstructorStruct(String name, int age) {\n" - + " constructorCalls++;\n" - + " this.name = name;\n" - + " this.age = age;\n" - + " }\n" - + " public String getName() { return name; }\n" - + " public int getAge() { return age; }\n" - + "}\n"); - Assert.assertTrue(result.success, result.diagnostics()); - try (URLClassLoader loader = result.classLoader()) { - Class type = loader.loadClass("test.RegisteredConstructorStruct"); - Constructor constructor = type.getConstructor(String.class, int.class); - Object value = constructor.newInstance("registered", 34); - setField(type, value, "note", "ctor"); - - Fory fory = - Fory.builder() - .withXlang(false) - .withClassLoader(loader) - .withCodegen(false) - .requireClassRegistration(false) - .build(); - fory.registerConstructor(type, constructor, "name", "age"); - Object serializer = fory.getTypeResolver().getTypeInfo(type).getSerializer(); - Assert.assertTrue(serializer instanceof StaticGeneratedStructSerializer); - - setField(type, null, "constructorCalls", 0); - Object roundTrip = fory.deserialize(fory.serialize(value)); - Assert.assertEquals(getField(type, null, "constructorCalls"), 1); - Assert.assertEquals(invoke(type, roundTrip, "getName"), "registered"); - Assert.assertEquals(invoke(type, roundTrip, "getAge"), 34); - Assert.assertEquals(getField(type, roundTrip, "note"), "ctor"); - } - } - - @Test - public void testStaticCtorCopyBackref() throws Exception { - CompilationResult result = - compile( - "test.StaticCtorBackref", - "package test;\n" - + "import org.apache.fory.annotation.ForyConstructor;\n" - + "import org.apache.fory.annotation.ForyStruct;\n" - + "@ForyStruct public class StaticCtorBackref {\n" - + " public Object self;\n" - + " @ForyConstructor(\"self\")\n" - + " public StaticCtorBackref(Object self) { this.self = self; }\n" - + "}\n"); - Assert.assertTrue(result.success, result.diagnostics()); - String generatedSource = - result.generatedSource("test/StaticCtorBackref_ForyNativeSerializer.java"); - Assert.assertTrue(generatedSource.contains("beginConstructorCopy(copyContext, value)")); - Assert.assertTrue(generatedSource.contains("checkNoConstructorCopyBackrefs(")); - try (URLClassLoader loader = result.classLoader()) { - Class type = loader.loadClass("test.StaticCtorBackref"); - Object value = type.getConstructor(Object.class).newInstance((Object) null); - setField(type, value, "self", value); - Fory fory = - Fory.builder() - .withXlang(false) - .withClassLoader(loader) - .withCodegen(false) - .withRefTracking(true) - .withRefCopy(true) - .requireClassRegistration(false) - .build(); - Object serializer = fory.getTypeResolver().getTypeInfo(type).getSerializer(); - Assert.assertTrue(serializer instanceof StaticGeneratedStructSerializer); - Assert.assertThrows(ForyException.class, () -> fory.copy(value)); - } - } - @Test public void testForyDebugAnnotationEmitsGeneratedFieldTracing() throws Exception { CompilationResult result = diff --git a/java/fory-core/pom.xml b/java/fory-core/pom.xml index e6b7ea8380..ed946e64ff 100644 --- a/java/fory-core/pom.xml +++ b/java/fory-core/pom.xml @@ -593,13 +593,18 @@ + org.apache.fory.core. JDK11 requires an update input file; re-adding an + unchanged base class avoids validating versioned classes while preserving + the descriptor refresh performed by the jar module-version option. --> + + + diff --git a/java/fory-core/src/main/java/org/apache/fory/AbstractThreadSafeFory.java b/java/fory-core/src/main/java/org/apache/fory/AbstractThreadSafeFory.java index 9f93cc21dc..8070a7d1cf 100644 --- a/java/fory-core/src/main/java/org/apache/fory/AbstractThreadSafeFory.java +++ b/java/fory-core/src/main/java/org/apache/fory/AbstractThreadSafeFory.java @@ -19,11 +19,8 @@ package org.apache.fory; -import java.lang.reflect.Constructor; import java.util.function.Function; -import org.apache.fory.exception.ForyException; import org.apache.fory.resolver.TypeChecker; -import org.apache.fory.resolver.TypeInfo; import org.apache.fory.resolver.TypeResolver; import org.apache.fory.serializer.Serializer; import org.apache.fory.serializer.SerializerFactory; @@ -69,31 +66,6 @@ public void register(ForyModule module) { registerCallback(fory -> fory.register(module)); } - @Override - public void registerConstructor( - Class type, Constructor constructor, String... fieldNames) { - String[] copiedFieldNames = fieldNames.clone(); - registerCallback(fory -> fory.registerConstructor(type, constructor, copiedFieldNames)); - } - - protected static void checkRegisterConstructorAllowed(Fory fory, Class type) { - TypeResolver typeResolver = fory.getTypeResolver(); - if (typeResolver.isRegistrationFinished()) { - throw new ForyException( - "Cannot register class/serializer after registration has been frozen. Please register " - + "all classes before invoking top-level `serialize/deserialize/copy` methods of " - + "Fory."); - } - TypeInfo typeInfo = typeResolver.getTypeInfo(type, false); - if (typeInfo != null && typeInfo.getSerializer() != null) { - throw new ForyException( - "Cannot register constructor for " - + type.getName() - + " after its serializer has been created. Register constructors before calling " - + "`getSerializer`, `serialize`, `deserialize`, or `copy` for that type."); - } - } - public void registerUnion( Class cls, int id, org.apache.fory.serializer.Serializer serializer) { registerCallback(fory -> fory.registerUnion(cls, id, serializer)); diff --git a/java/fory-core/src/main/java/org/apache/fory/BaseFory.java b/java/fory-core/src/main/java/org/apache/fory/BaseFory.java index 6f32c34ffd..d214ac5334 100644 --- a/java/fory-core/src/main/java/org/apache/fory/BaseFory.java +++ b/java/fory-core/src/main/java/org/apache/fory/BaseFory.java @@ -20,7 +20,6 @@ package org.apache.fory; import java.io.OutputStream; -import java.lang.reflect.Constructor; import java.nio.ByteBuffer; import java.util.function.Function; import org.apache.fory.io.ForyInputStream; @@ -90,20 +89,6 @@ public interface BaseFory { */ void register(ForyModule module); - /** - * Register an explicit constructor-to-field mapping for {@code type}. - * - *

    The constructor arguments are populated from {@code fieldNames} in order. This is useful for - * third-party classes that cannot annotate a constructor with {@code @ForyConstructor}. {@code - * fieldNames} must contain at least one field; ordinary no-argument constructors should not be - * registered because they do not need constructor-to-field binding. Java platform classes are - * owned by built-in serializers and cannot use explicit constructor binding. - * - *

    Call this during setup before serializers for {@code type} are requested and before - * top-level serialization, deserialization, or copy operations start. - */ - void registerConstructor(Class type, Constructor constructor, String... fieldNames); - void registerUnion(Class cls, int id, Serializer serializer); void registerUnion(Class cls, String namespace, String typeName, Serializer serializer); diff --git a/java/fory-core/src/main/java/org/apache/fory/Fory.java b/java/fory-core/src/main/java/org/apache/fory/Fory.java index 77d32c0ba1..39f4890c87 100644 --- a/java/fory-core/src/main/java/org/apache/fory/Fory.java +++ b/java/fory-core/src/main/java/org/apache/fory/Fory.java @@ -21,7 +21,6 @@ import java.io.IOException; import java.io.OutputStream; -import java.lang.reflect.Constructor; import java.nio.ByteBuffer; import java.util.IdentityHashMap; import java.util.function.Consumer; @@ -212,21 +211,6 @@ public void register(ForyModule module) { } } - @Override - public void registerConstructor( - Class type, Constructor constructor, String... fieldNames) { - checkRegisterAllowed(); - TypeInfo typeInfo = typeResolver.getTypeInfo(type, false); - if (typeInfo != null && typeInfo.getSerializer() != null) { - throw new ForyException( - "Cannot register constructor for " - + type.getName() - + " after its serializer has been created. Register constructors before calling " - + "`getSerializer`, `serialize`, `deserialize`, or `copy` for that type."); - } - sharedRegistry.registerConstructor(type, constructor, fieldNames); - } - @Override public void registerUnion(Class cls, int id, Serializer serializer) { getTypeResolver().registerUnion(cls, Integer.toUnsignedLong(id), serializer); diff --git a/java/fory-core/src/main/java/org/apache/fory/ThreadLocalFory.java b/java/fory-core/src/main/java/org/apache/fory/ThreadLocalFory.java index 2a2422332f..57f0d947dc 100644 --- a/java/fory-core/src/main/java/org/apache/fory/ThreadLocalFory.java +++ b/java/fory-core/src/main/java/org/apache/fory/ThreadLocalFory.java @@ -20,7 +20,6 @@ package org.apache.fory; import java.io.OutputStream; -import java.lang.reflect.Constructor; import java.nio.ByteBuffer; import java.util.Collections; import java.util.Map; @@ -87,20 +86,6 @@ public void registerCallback(Consumer callback) { } } - @Override - public void registerConstructor( - Class type, Constructor constructor, String... fieldNames) { - String[] copiedFieldNames = fieldNames.clone(); - Consumer callback = fory -> fory.registerConstructor(type, constructor, copiedFieldNames); - synchronized (callbackLock) { - synchronized (allFory) { - allFory.keySet().forEach(fory -> checkRegisterConstructorAllowed(fory, type)); - allFory.keySet().forEach(callback); - } - factoryCallback = factoryCallback.andThen(callback); - } - } - @Override public R execute(Function action) { return action.apply(currentFory()); diff --git a/java/fory-core/src/main/java/org/apache/fory/annotation/ForyConstructor.java b/java/fory-core/src/main/java/org/apache/fory/annotation/ForyConstructor.java deleted file mode 100644 index ee5053434b..0000000000 --- a/java/fory-core/src/main/java/org/apache/fory/annotation/ForyConstructor.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.fory.annotation; - -import java.lang.annotation.Documented; -import java.lang.annotation.ElementType; -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; -import java.lang.annotation.Target; - -/** Maps one constructor's arguments to serialized field names. */ -@Documented -@Retention(RetentionPolicy.RUNTIME) -@Target(ElementType.CONSTRUCTOR) -@Public -public @interface ForyConstructor { - /** - * Field names in constructor argument order. - * - *

    At least one field must be listed. Ordinary no-argument constructors should not be annotated - * because they do not need constructor-to-field binding. - * - *

    Every name must refer to one non-static serialized field declared by the target class or a - * superclass. Duplicate field names in a class hierarchy are not bindable by this annotation. - */ - String[] value(); -} diff --git a/java/fory-core/src/main/java/org/apache/fory/builder/CompatibleCodecBuilder.java b/java/fory-core/src/main/java/org/apache/fory/builder/CompatibleCodecBuilder.java index 676b7a999a..45006d2ff7 100644 --- a/java/fory-core/src/main/java/org/apache/fory/builder/CompatibleCodecBuilder.java +++ b/java/fory-core/src/main/java/org/apache/fory/builder/CompatibleCodecBuilder.java @@ -42,7 +42,6 @@ import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.meta.TypeDef; import org.apache.fory.platform.GraalvmSupport; -import org.apache.fory.reflect.ObjectCreator; import org.apache.fory.reflect.TypeRef; import org.apache.fory.resolver.TypeResolver; import org.apache.fory.serializer.CodegenSerializer; @@ -131,29 +130,6 @@ public CompatibleCodecBuilder(TypeRef beanType, Fory fory, TypeDef typeDef) { } this.defaultValueLanguage = defaultValueLanguage; this.defaultValueFields = defaultValueFields; - if (!isRecord) { - initConstructorFields( - sortedDescriptors, - true, - defaultFieldNames(defaultValueFields), - defaultDeclaringClasses(defaultValueFields)); - } - } - - private static String[] defaultFieldNames(DefaultValueUtils.DefaultValueField[] fields) { - String[] names = new String[fields.length]; - for (int i = 0; i < fields.length; i++) { - names[i] = fields[i].getFieldName(); - } - return names; - } - - private static Class[] defaultDeclaringClasses(DefaultValueUtils.DefaultValueField[] fields) { - Class[] declaringClasses = new Class[fields.length]; - for (int i = 0; i < fields.length; i++) { - declaringClasses[i] = fields[i].getDeclaringClass(); - } - return declaringClasses; } // Must be static to be shared across the whole process life. @@ -324,20 +300,11 @@ protected Expression newBean() { return setDefaultsExpr; } - @Override - protected void postCreateConstructorObject( - Expression.ListExpression expressions, Expression bean) { - addDefaultValueSetters(expressions, bean); - } - private void addDefaultValueSetters(Expression.ListExpression expressions, Expression bean) { Map descriptors = Descriptor.getAllDescriptorsMap(beanClass); for (DefaultValueUtils.DefaultValueField defaultField : defaultValueFields) { Object defaultValue = defaultField.getDefaultValue(); Member member = defaultField.getFieldAccessor().getField(); - if (constructorOwnsField(member)) { - continue; - } Descriptor descriptor = descriptors.get(member); TypeRef typeRef = descriptor.getTypeRef(); Expression defaultValueExpr; @@ -364,51 +331,4 @@ private void addDefaultValueSetters(Expression.ListExpression expressions, Expre expressions.add(super.setFieldValue(bean, descriptor, defaultValueExpr)); } } - - private boolean constructorOwnsField(Member member) { - if (constructorFieldIndexes == null) { - return false; - } - ObjectCreator objectCreator = typeResolver.getObjectCreator(beanClass); - String[] names = objectCreator.getConstructorFieldNames(); - Class[] declaringClasses = objectCreator.getConstructorFieldDeclaringClasses(); - for (int i = 0; i < names.length; i++) { - Class declaringClass = declaringClasses == null ? null : declaringClasses[i]; - if (names[i].equals(member.getName()) - && (declaringClass == null || declaringClass == member.getDeclaringClass())) { - return true; - } - } - return false; - } - - @Override - protected Expression defaultConstructorValue(int constructorParameterIndex) { - ObjectCreator objectCreator = typeResolver.getObjectCreator(beanClass); - String fieldName = objectCreator.getConstructorFieldNames()[constructorParameterIndex]; - Class[] declaringClasses = objectCreator.getConstructorFieldDeclaringClasses(); - Class declaringClass = - declaringClasses == null ? null : declaringClasses[constructorParameterIndex]; - for (DefaultValueUtils.DefaultValueField defaultField : defaultValueFields) { - if (!defaultField.getFieldName().equals(fieldName) - || (declaringClass != null && defaultField.getDeclaringClass() != declaringClass)) { - continue; - } - Object defaultValue = defaultField.getDefaultValue(); - TypeRef typeRef = TypeRef.of(constructorFieldTypes[constructorParameterIndex]); - if (typeRef.unwrap().isPrimitive() || typeRef.equals(STRING_TYPE)) { - return new Literal(defaultValue, typeRef); - } - String funcName = "get" + defaultValueLanguage + "DefaultValue"; - return new Expression.Cast( - new StaticInvoke( - DefaultValueUtils.class, - funcName, - OBJECT_TYPE, - staticBeanClassExpr(), - Literal.ofString(fieldName)), - typeRef); - } - return super.defaultConstructorValue(constructorParameterIndex); - } } diff --git a/java/fory-core/src/main/java/org/apache/fory/builder/ObjectCodecBuilder.java b/java/fory-core/src/main/java/org/apache/fory/builder/ObjectCodecBuilder.java index 04bce6bce3..026133fe8e 100644 --- a/java/fory-core/src/main/java/org/apache/fory/builder/ObjectCodecBuilder.java +++ b/java/fory-core/src/main/java/org/apache/fory/builder/ObjectCodecBuilder.java @@ -41,7 +41,6 @@ import java.util.ArrayList; import java.util.Collection; -import java.util.IdentityHashMap; import java.util.List; import java.util.Map; import java.util.Objects; @@ -65,9 +64,7 @@ import org.apache.fory.logging.LoggerFactory; import org.apache.fory.meta.TypeDef; import org.apache.fory.platform.JdkVersion; -import org.apache.fory.reflect.ObjectCreator; import org.apache.fory.reflect.TypeRef; -import org.apache.fory.serializer.AbstractObjectSerializer; import org.apache.fory.serializer.ObjectSerializer; import org.apache.fory.type.BFloat16; import org.apache.fory.type.Descriptor; @@ -100,10 +97,6 @@ public class ObjectCodecBuilder extends BaseObjectCodecBuilder { private final Literal classVersionHash; protected ObjectCodecOptimizer objectCodecOptimizer; protected Map recordReversedMapping; - protected Map fieldIndexes; - protected int[] constructorFieldIndexes; - protected boolean[] constructorFieldMask; - protected Class[] constructorFieldTypes; public ObjectCodecBuilder(Class beanClass, Fory fory) { super(TypeRef.of(beanClass), fory, Generated.GeneratedObjectSerializer.class); @@ -144,8 +137,6 @@ public ObjectCodecBuilder(Class beanClass, Fory fory) { buildRecordComponentDefaultValues(); } recordReversedMapping = RecordUtils.buildFieldToComponentMapping(beanClass); - } else { - initConstructorFields(grouper.getSortedDescriptors(), true); } } @@ -160,122 +151,6 @@ protected ObjectCodecBuilder(TypeRef beanType, Fory fory, Class superSeria } } - protected final void initConstructorFields( - List sortedDescriptors, boolean allowMissingNonFinal) { - initConstructorFields(sortedDescriptors, allowMissingNonFinal, null); - } - - protected final void initConstructorFields( - List sortedDescriptors, boolean allowMissingNonFinal, String[] defaultFields) { - initConstructorFields(sortedDescriptors, allowMissingNonFinal, defaultFields, null); - } - - protected final void initConstructorFields( - List sortedDescriptors, - boolean allowMissingNonFinal, - String[] defaultFields, - Class[] defaultDeclaringClasses) { - ObjectCreator objectCreator = typeResolver.getObjectCreator(beanClass); - if (!objectCreator.hasConstructorFields()) { - return; - } - fieldIndexes = buildFieldIndexes(sortedDescriptors); - constructorFieldTypes = objectCreator.getConstructorFieldTypes(); - constructorFieldIndexes = - buildConstructorFieldIndexes( - sortedDescriptors, - objectCreator, - allowMissingNonFinal, - defaultFields, - defaultDeclaringClasses); - constructorFieldMask = buildConstructorFieldMask(sortedDescriptors.size()); - } - - private static Map buildFieldIndexes(List descriptors) { - Map indexes = new IdentityHashMap<>(); - for (int i = 0; i < descriptors.size(); i++) { - indexes.put(descriptors.get(i), i); - } - return indexes; - } - - private int[] buildConstructorFieldIndexes( - List descriptors, - ObjectCreator objectCreator, - boolean allowMissingNonFinal, - String[] defaultFields, - Class[] defaultDeclaringClasses) { - String[] names = objectCreator.getConstructorFieldNames(); - Class[] declaringClasses = objectCreator.getConstructorFieldDeclaringClasses(); - boolean[] finalFields = objectCreator.getConstructorFieldFinal(); - int[] indexes = new int[names.length]; - for (int i = 0; i < names.length; i++) { - Class declaringClass = declaringClasses == null ? null : declaringClasses[i]; - boolean allowMissing = - (allowMissingNonFinal && !finalFields[i]) - || contains(defaultFields, defaultDeclaringClasses, names[i], declaringClass); - indexes[i] = constructorFieldIndex(descriptors, declaringClass, names[i], allowMissing); - } - return indexes; - } - - private static boolean contains( - String[] values, Class[] declaringClasses, String value, Class declaringClass) { - if (values == null) { - return false; - } - for (int i = 0; i < values.length; i++) { - if (values[i].equals(value) - && (declaringClasses == null - || i >= declaringClasses.length - || declaringClasses[i] == null - || declaringClasses[i] == declaringClass)) { - return true; - } - } - return false; - } - - private int constructorFieldIndex( - List descriptors, - Class declaringClass, - String fieldName, - boolean allowMissing) { - int index = -1; - for (int i = 0; i < descriptors.size(); i++) { - Descriptor descriptor = descriptors.get(i); - if (!descriptor.getName().equals(fieldName) - || (declaringClass != null - && (descriptor.getField() == null - || descriptor.getField().getDeclaringClass() != declaringClass))) { - continue; - } - if (index >= 0) { - throw new IllegalStateException( - "Constructor field " + fieldName + " is ambiguous for " + beanClass); - } - index = i; - } - if (index < 0) { - if (allowMissing) { - return -1; - } - throw new IllegalStateException( - "Constructor field " + fieldName + " is not serialized for " + beanClass); - } - return index; - } - - private boolean[] buildConstructorFieldMask(int size) { - boolean[] mask = new boolean[size]; - for (int index : constructorFieldIndexes) { - if (index >= 0) { - mask[index] = true; - } - } - return mask; - } - @Override protected String codecSuffix() { return ""; @@ -1001,20 +876,12 @@ public Expression buildDecodeExpression() { if (typeResolver.checkClassVersion()) { expressions.add(checkClassVersion(buffer)); } - if (!isRecord && constructorFieldIndexes != null) { - return buildConstructorDecodeExpression(buffer, expressions); - } Expression bean; if (!isRecord) { - if (constructorFieldIndexes == null) { - bean = newBean(); - Expression referenceObject = invokeReadContext("reference", bean); - expressions.add(bean); - expressions.add(referenceObject); - } else { - bean = new FieldsArray(fieldIndexes.size()); - expressions.add(bean); - } + bean = newBean(); + Expression referenceObject = invokeReadContext("reference", bean); + expressions.add(bean); + expressions.add(referenceObject); } else { if (recordCtrAccessible) { bean = new FieldsCollector(); @@ -1043,147 +910,6 @@ public Expression buildDecodeExpression() { return expressions; } - private Expression buildConstructorDecodeExpression( - Reference buffer, ListExpression expressions) { - FieldsArray fieldsArray = new FieldsArray(fieldIndexes.size()); - expressions.add(fieldsArray); - expressions.add( - new StaticInvoke( - AbstractObjectSerializer.class, - "beginConstructorRef", - PRIMITIVE_VOID_TYPE, - readContextRef())); - List bufferedNonConstructorFields = new ArrayList<>(); - int remainingConstructorFields = countConstructorFields(); - Expression bean = null; - if (remainingConstructorFields == 0) { - bean = createCtorBean(expressions, fieldsArray); - } - for (Descriptor descriptor : protocolDescriptors()) { - int index = fieldIndexes.get(descriptor); - walkPath.add(descriptor.getDeclaringClass() + descriptor.getName()); - if (constructorFieldMask[index]) { - trackConstructorRefRead(expressions, buffer, descriptor); - expressions.add(deserializeToFieldsArray(fieldsArray, buffer, descriptor, true)); - remainingConstructorFields--; - if (remainingConstructorFields == 0) { - bean = createCtorBean(expressions, fieldsArray); - addBufferedFieldSetters(expressions, bean, fieldsArray, bufferedNonConstructorFields); - } - } else if (bean == null) { - trackConstructorRefRead(expressions, buffer, descriptor); - expressions.add(deserializeToFieldsArray(fieldsArray, buffer, descriptor, false)); - bufferedNonConstructorFields.add(descriptor); - } else { - expressions.add(deserializeToBean(bean, buffer, descriptor)); - } - walkPath.removeLast(); - } - expressions.add( - new StaticInvoke( - AbstractObjectSerializer.class, - "endConstructorRef", - PRIMITIVE_VOID_TYPE, - readContextRef())); - expressions.add(new Expression.Return(bean)); - return expressions; - } - - private int countConstructorFields() { - int count = 0; - for (boolean constructorField : constructorFieldMask) { - if (constructorField) { - count++; - } - } - return count; - } - - private List protocolDescriptors() { - List descriptors = new ArrayList<>(); - addDescriptors(descriptors, objectCodecOptimizer.primitiveGroups); - addDescriptors(descriptors, objectCodecOptimizer.boxedReadGroups); - addDescriptors(descriptors, objectCodecOptimizer.nonPrimitiveReadGroups); - return descriptors; - } - - private void trackConstructorRefRead( - ListExpression expressions, Reference buffer, Descriptor descriptor) { - if (descriptor.isTrackingRef()) { - expressions.add( - new StaticInvoke( - AbstractObjectSerializer.class, - "trackConstructorRefRead", - PRIMITIVE_VOID_TYPE, - readContextRef(), - buffer)); - } - } - - private void addDescriptors(List descriptors, List> groups) { - for (List group : groups) { - descriptors.addAll(group); - } - } - - private Expression createCtorBean(ListExpression expressions, FieldsArray fieldsArray) { - Expression bean = createConstructorObject(fieldsArray); - expressions.add( - new StaticInvoke( - AbstractObjectSerializer.class, - "checkNoUnresolvedReadRef", - PRIMITIVE_VOID_TYPE, - readContextRef(), - staticBeanClassExpr())); - expressions.add(bean); - expressions.add( - new StaticInvoke( - AbstractObjectSerializer.class, - "referenceConstructorRef", - PRIMITIVE_VOID_TYPE, - readContextRef(), - bean)); - postCreateConstructorObject(expressions, bean); - return bean; - } - - private Expression deserializeToFieldsArray( - FieldsArray fieldsArray, Reference buffer, Descriptor descriptor, boolean constructorField) { - TypeRef castTypeRef = - hasCompatibleCollectionArrayRead(descriptor) - ? compatibleReadTargetTypeRef(descriptor) - : descriptor.getTypeRef(); - return deserializeField( - buffer, - descriptor, - expr -> { - Expression value = - constructorField ? tryInlineCast(expr, castTypeRef) : new Cast(expr, OBJECT_TYPE); - value = - new StaticInvoke( - AbstractObjectSerializer.class, - constructorField ? "ctorFieldValue" : "bufferFieldValue", - OBJECT_TYPE, - readContextRef(), - value, - staticBeanClassExpr()); - return setFieldValue(fieldsArray, descriptor, value); - }); - } - - private Expression deserializeToBean(Expression bean, Reference buffer, Descriptor descriptor) { - TypeRef castTypeRef = - hasCompatibleCollectionArrayRead(descriptor) - ? compatibleReadTargetTypeRef(descriptor) - : descriptor.getTypeRef(); - return deserializeField( - buffer, - descriptor, - expr -> setFieldValue(bean, descriptor, tryInlineCast(expr, castTypeRef))); - } - - protected void postCreateConstructorObject(ListExpression expressions, Expression bean) {} - protected void deserializeReadGroup( List> readGroups, int numGroups, @@ -1209,105 +935,6 @@ protected Expression createRecord(SortedMap recordComponent return new NewInstance(beanType, params); } - protected Expression createConstructorObject(FieldsArray fieldValues) { - Expression[] params = new Expression[constructorFieldIndexes.length]; - Expression[] directParams = new Expression[constructorFieldIndexes.length]; - for (int i = 0; i < constructorFieldIndexes.length; i++) { - int index = constructorFieldIndexes[i]; - if (index < 0) { - params[i] = defaultConstructorValue(i); - } else { - params[i] = fieldValue(fieldValues, index); - } - directParams[i] = tryInlineCast(params[i], TypeRef.of(constructorFieldTypes[i])); - } - ObjectCreator objectCreator = typeResolver.getObjectCreator(beanClass); - if (JdkVersion.MAJOR_VERSION >= 25 - && objectCreator.isOnlyPublicConstructor() - && sourcePublicAccessible(beanClass) - && constructorParamsAccessible()) { - return new NewInstance(beanType, directParams); - } - Expression args = new Expression.NewArray(OBJECT_ARRAY_TYPE, params); - Expression newInstance = - new Invoke(getObjectCreator(beanClass), "newInstanceWithArguments", OBJECT_TYPE, args); - return sourcePublicAccessible(beanClass) ? new Cast(newInstance, beanType) : newInstance; - } - - protected Expression defaultConstructorValue(int constructorParameterIndex) { - return new StaticInvoke( - AbstractObjectSerializer.class, - "defaultConstructorValue", - OBJECT_TYPE, - staticClassFieldExpr( - constructorFieldTypes[constructorParameterIndex], - "constructorFieldClass" + constructorParameterIndex + "_")); - } - - private boolean constructorParamsAccessible() { - for (Class constructorFieldType : constructorFieldTypes) { - if (!sourcePublicAccessible(constructorFieldType)) { - return false; - } - } - return true; - } - - private void addNonConstructorFieldSetters( - ListExpression expressions, Expression bean, FieldsArray fieldValues) { - for (Descriptor descriptor : objectCodecOptimizer.descriptorGrouper.getSortedDescriptors()) { - int index = fieldIndexes.get(descriptor); - if (constructorFieldMask[index]) { - continue; - } - TypeRef castTypeRef = - hasCompatibleCollectionArrayRead(descriptor) - ? compatibleReadTargetTypeRef(descriptor) - : descriptor.getTypeRef(); - Expression value = - new StaticInvoke( - AbstractObjectSerializer.class, - "resolveBufferedValue", - OBJECT_TYPE, - fieldValue(fieldValues, index), - bean); - value = tryInlineCast(value, castTypeRef); - expressions.add(setFieldValue(bean, descriptor, value)); - } - } - - private void addBufferedFieldSetters( - ListExpression expressions, - Expression bean, - FieldsArray fieldValues, - List descriptors) { - for (Descriptor descriptor : descriptors) { - int index = fieldIndexes.get(descriptor); - TypeRef castTypeRef = - hasCompatibleCollectionArrayRead(descriptor) - ? compatibleReadTargetTypeRef(descriptor) - : descriptor.getTypeRef(); - Expression value = - new StaticInvoke( - AbstractObjectSerializer.class, - "resolveBufferedValue", - OBJECT_TYPE, - fieldValue(fieldValues, index), - bean); - value = tryInlineCast(value, castTypeRef); - expressions.add(setFieldValue(bean, descriptor, value)); - } - } - - private Expression fieldValue(Expression fieldValues, int index) { - return new StaticInvoke( - AbstractObjectSerializer.class, - "fieldValue", - OBJECT_TYPE, - fieldValues, - Literal.ofInt(index)); - } - private class FieldsCollector extends Expression.AbstractExpression { private final TreeMap recordValuesMap = new TreeMap<>(); @@ -1326,38 +953,8 @@ public Code.ExprCode doGenCode(CodegenContext ctx) { } } - protected class FieldsArray extends Expression.AbstractExpression { - private final int size; - private final String name; - - protected FieldsArray(int size) { - super(new Expression[0]); - this.size = size; - name = ctx.newName("fieldValues"); - } - - @Override - public TypeRef type() { - return OBJECT_ARRAY_TYPE; - } - - @Override - public Code.ExprCode doGenCode(CodegenContext ctx) { - String code = ctx.type(Object[].class) + " " + name + " = new Object[" + size + "];"; - return new Code.ExprCode(code, FalseLiteral, Code.variable(Object[].class, name)); - } - - int fieldIndex(Descriptor descriptor) { - return fieldIndexes.get(descriptor); - } - } - @Override protected Expression setFieldValue(Expression bean, Descriptor d, Expression value) { - if (bean instanceof FieldsArray) { - return new Expression.AssignArrayElem( - bean, value, Literal.ofInt(((FieldsArray) bean).fieldIndex(d))); - } if (isRecord) { if (recordCtrAccessible) { if (value instanceof Inlineable) { diff --git a/java/fory-core/src/main/java/org/apache/fory/context/ReadContext.java b/java/fory-core/src/main/java/org/apache/fory/context/ReadContext.java index 6dca2e503a..79398e2293 100644 --- a/java/fory-core/src/main/java/org/apache/fory/context/ReadContext.java +++ b/java/fory-core/src/main/java/org/apache/fory/context/ReadContext.java @@ -30,6 +30,7 @@ import org.apache.fory.resolver.TypeInfo; import org.apache.fory.resolver.TypeInfoHolder; import org.apache.fory.resolver.TypeResolver; +import org.apache.fory.serializer.AbstractObjectSerializer; import org.apache.fory.serializer.PrimitiveSerializers.LongSerializer; import org.apache.fory.serializer.Serializer; import org.apache.fory.serializer.StringSerializer; @@ -342,6 +343,12 @@ public int preserveRefId(int refId) { /** Delegates to {@link RefReader#tryPreserveRefId(MemoryBuffer)} on the current buffer. */ public int tryPreserveRefId() { + if (refReader.hasPreservedRefId()) { + // Constructor-bound objects cannot satisfy self-references until construction finishes. + // The guard keeps ordinary ref reads on the direct path after standard serializers bind + // their object before reading fields. + AbstractObjectSerializer.trackConstructorRefRead(this, buffer); + } return refReader.tryPreserveRefId(buffer); } @@ -492,7 +499,7 @@ public String readString() { public String readStringRef() { MemoryBuffer buffer = this.buffer; if (stringSerializer.needToWriteRef()) { - int nextReadRefId = refReader.tryPreserveRefId(buffer); + int nextReadRefId = tryPreserveRefId(); if (nextReadRefId >= Fory.NOT_NULL_VALUE_FLAG) { String obj = stringSerializer.read(this); refReader.setReadRef(nextReadRefId, obj); @@ -526,8 +533,7 @@ public long readInt64() { * directly. */ public Object readRef() { - MemoryBuffer buffer = this.buffer; - int nextReadRefId = refReader.tryPreserveRefId(buffer); + int nextReadRefId = tryPreserveRefId(); if (nextReadRefId >= Fory.NOT_NULL_VALUE_FLAG) { TypeInfo typeInfo = typeResolver.readTypeInfo(this); Object o = readNonRef(typeInfo); @@ -539,7 +545,7 @@ public Object readRef() { /** Variant of {@link #readRef()} that uses already resolved {@link TypeInfo}. */ public Object readRef(TypeInfo typeInfo) { - int nextReadRefId = refReader.tryPreserveRefId(buffer); + int nextReadRefId = tryPreserveRefId(); if (nextReadRefId >= Fory.NOT_NULL_VALUE_FLAG) { Object o = readNonRef(typeInfo); refReader.setReadRef(nextReadRefId, o); @@ -550,7 +556,7 @@ public Object readRef(TypeInfo typeInfo) { /** Variant of {@link #readRef()} that reuses a cached type-info holder. */ public Object readRef(TypeInfoHolder classInfoHolder) { - int nextReadRefId = refReader.tryPreserveRefId(buffer); + int nextReadRefId = tryPreserveRefId(); if (nextReadRefId >= Fory.NOT_NULL_VALUE_FLAG) { TypeInfo typeInfo = typeResolver.readTypeInfo(this, classInfoHolder); Object o = readNonRef(typeInfo); @@ -563,7 +569,7 @@ public Object readRef(TypeInfoHolder classInfoHolder) { /** Reads a nullable object using an already chosen serializer. */ public T readRef(Serializer serializer) { if (serializer.needToWriteRef()) { - int nextReadRefId = refReader.tryPreserveRefId(buffer); + int nextReadRefId = tryPreserveRefId(); if (nextReadRefId >= Fory.NOT_NULL_VALUE_FLAG) { Object o = readNonRef(serializer); refReader.setReadRef(nextReadRefId, o); diff --git a/java/fory-core/src/main/java/org/apache/fory/pool/ThreadPoolFory.java b/java/fory-core/src/main/java/org/apache/fory/pool/ThreadPoolFory.java index 2bba3c3e74..e3544aa106 100644 --- a/java/fory-core/src/main/java/org/apache/fory/pool/ThreadPoolFory.java +++ b/java/fory-core/src/main/java/org/apache/fory/pool/ThreadPoolFory.java @@ -20,7 +20,6 @@ package org.apache.fory.pool; import java.io.OutputStream; -import java.lang.reflect.Constructor; import java.nio.ByteBuffer; import java.util.concurrent.Semaphore; import java.util.concurrent.atomic.AtomicInteger; @@ -155,20 +154,6 @@ public void registerCallback(Consumer callback) { } } - @Override - public void registerConstructor( - Class type, Constructor constructor, String... fieldNames) { - String[] copiedFieldNames = fieldNames.clone(); - synchronized (callbackLock) { - for (Fory fory : pooledFory) { - checkRegisterConstructorAllowed(fory, type); - } - for (Fory fory : pooledFory) { - fory.registerConstructor(type, constructor, copiedFieldNames); - } - } - } - @Override public R execute(Function action) { PooledEntry entry = acquire(); diff --git a/java/fory-core/src/main/java/org/apache/fory/reflect/ObjectCreator.java b/java/fory-core/src/main/java/org/apache/fory/reflect/ObjectCreator.java index 1134c21896..0335a63af0 100644 --- a/java/fory-core/src/main/java/org/apache/fory/reflect/ObjectCreator.java +++ b/java/fory-core/src/main/java/org/apache/fory/reflect/ObjectCreator.java @@ -37,10 +37,6 @@ */ @ThreadSafe public abstract class ObjectCreator { - private static final String[] NO_FIELDS = new String[0]; - private static final Class[] NO_TYPES = new Class[0]; - private static final boolean[] NO_FINAL_FIELDS = new boolean[0]; - protected final Class type; protected ObjectCreator(Class type) { @@ -56,34 +52,6 @@ protected ObjectCreator(Class type) { */ public abstract T newInstance(); - public boolean hasConstructorFields() { - return false; - } - - public String[] getConstructorFieldNames() { - return NO_FIELDS; - } - - public Class[] getConstructorFieldDeclaringClasses() { - return null; - } - - public Class[] getConstructorFieldTypes() { - return NO_TYPES; - } - - public boolean[] getConstructorFieldFinal() { - return NO_FINAL_FIELDS; - } - - public boolean isConstructorPublic() { - return false; - } - - public boolean isOnlyPublicConstructor() { - return false; - } - /** * Creates a new instance of type T using the provided arguments. * diff --git a/java/fory-core/src/main/java/org/apache/fory/reflect/ObjectCreators.java b/java/fory-core/src/main/java/org/apache/fory/reflect/ObjectCreators.java index 386ac77862..930e53fd70 100644 --- a/java/fory-core/src/main/java/org/apache/fory/reflect/ObjectCreators.java +++ b/java/fory-core/src/main/java/org/apache/fory/reflect/ObjectCreators.java @@ -21,21 +21,10 @@ import java.io.Serializable; import java.lang.invoke.MethodHandle; -import java.lang.invoke.MethodHandles.Lookup; -import java.lang.invoke.MethodType; import java.lang.reflect.Constructor; -import java.lang.reflect.Field; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.lang.reflect.Modifier; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.LinkedHashMap; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import org.apache.fory.annotation.ForyConstructor; import org.apache.fory.annotation.Internal; import org.apache.fory.collection.ClassValueCache; import org.apache.fory.collection.Tuple2; @@ -43,9 +32,6 @@ import org.apache.fory.platform.AndroidSupport; import org.apache.fory.platform.GraalvmSupport; import org.apache.fory.platform.JdkVersion; -import org.apache.fory.platform.internal._JDKAccess; -import org.apache.fory.type.Descriptor; -import org.apache.fory.type.TypeUtils; import org.apache.fory.util.record.RecordUtils; /** @@ -68,8 +54,8 @@ * *

    The static {@link #getObjectCreator(Class)} method keeps the legacy process-global cache. * Runtime-owned paths should use {@link - * org.apache.fory.resolver.TypeResolver#getObjectCreator(Class)} so constructor registrations and - * ObjectStream-compatible creators stay scoped to the Fory runtime. + * org.apache.fory.resolver.TypeResolver#getObjectCreator(Class)} so ObjectStream-compatible + * creators stay scoped to the Fory runtime. * *

    Thread Safety: This class and all returned ObjectCreator instances are * thread-safe and can be safely used across multiple threads concurrently. @@ -93,24 +79,15 @@ public class ObjectCreators { * GraalVM native image) */ public static ObjectCreator getObjectCreator(Class type) { - return (ObjectCreator) cache.get(type, () -> createObjectCreator(type, null)); + return (ObjectCreator) cache.get(type, () -> createObjectCreator(type)); } /** Creates an uncached object creator for runtime-scoped registries. */ @Internal public static ObjectCreator createObjectCreator(Class type) { - return createObjectCreator(type, null); - } - - static ObjectCreator createObjectCreator( - Class type, ConstructorMatch registeredConstructor) { if (RecordUtils.isRecord(type)) { return new RecordObjectCreator<>(type); } - ConstructorMatch explicitConstructor = explicitConstructor(type, registeredConstructor); - if (explicitConstructor != null) { - return new ConstructorObjectCreator<>(type, explicitConstructor); - } Constructor noArgConstructor = ReflectionUtils.getNoArgConstructor(type); if (AndroidSupport.IS_ANDROID) { if (noArgConstructor != null) { @@ -132,13 +109,6 @@ static ObjectCreator createObjectCreator( return new DeclaredNoArgCtrObjectCreator<>(type); } - /** Creates an uncached constructor-bound object creator for runtime-scoped registries. */ - @Internal - public static ObjectCreator createObjectCreator( - Class type, Constructor constructor, String[] fieldNames, String source) { - return createObjectCreator(type, explicitConstructor(type, constructor, fieldNames, source)); - } - /** Creates an uncached empty-instance creator for Java ObjectStream-compatible serializers. */ @Internal public static ObjectCreator createObjectStreamCreator(Class type) { @@ -156,192 +126,6 @@ public static ObjectCreator createObjectStreamCreator(Class type) { return new ParentNoArgCtrObjectCreator<>(type); } - static final class ConstructorMatch { - private final Constructor constructor; - private final String[] fieldNames; - private final Class[] declaringClasses; - private final Class[] fieldTypes; - private final boolean[] finalFields; - - private ConstructorMatch( - Constructor constructor, - String[] fieldNames, - Class[] declaringClasses, - Class[] fieldTypes, - boolean[] finalFields) { - this.constructor = constructor; - this.fieldNames = fieldNames; - this.declaringClasses = declaringClasses; - this.fieldTypes = fieldTypes; - this.finalFields = finalFields; - } - } - - private static ConstructorMatch explicitConstructor( - Class type, ConstructorMatch registeredConstructor) { - if (registeredConstructor != null) { - return registeredConstructor; - } - Constructor annotatedConstructor = null; - ForyConstructor annotation = null; - for (Constructor constructor : type.getDeclaredConstructors()) { - ForyConstructor currentAnnotation = constructor.getAnnotation(ForyConstructor.class); - if (currentAnnotation == null || isCompilerGeneratedConstructor(constructor)) { - continue; - } - if (annotatedConstructor != null) { - throw new ForyException(type + " must not declare more than one @ForyConstructor"); - } - annotatedConstructor = constructor; - annotation = currentAnnotation; - } - if (annotatedConstructor == null) { - return null; - } - return explicitConstructor( - type, (Constructor) annotatedConstructor, annotation.value(), "@ForyConstructor"); - } - - static ConstructorMatch explicitConstructor( - Class type, Constructor constructor, String[] fieldNames, String source) { - if (isJavaPlatformType(type)) { - throw new ForyException( - source - + " constructor binding is not supported for Java platform type " - + type.getName()); - } - if (constructor.getDeclaringClass() != type) { - throw new ForyException( - source + " constructor " + constructor + " does not belong to " + type); - } - if (fieldNames.length == 0) { - throw new ForyException( - source - + " constructor " - + constructor - + " must map at least one field. Leave ordinary no-arg constructors unbound."); - } - if (fieldNames.length != constructor.getParameterCount()) { - throw new ForyException( - source - + " constructor " - + constructor - + " maps " - + fieldNames.length - + " fields to " - + constructor.getParameterCount() - + " parameters"); - } - ConstructorMatch match = - matchConstructorFields( - constructor, fieldsByExplicitNames(type, constructor, fieldNames, source)); - if (match == null) { - throw new ForyException( - source - + " constructor " - + constructor - + " parameter types do not match fields " - + Arrays.toString(fieldNames)); - } - return match; - } - - private static boolean isCompilerGeneratedConstructor(Constructor constructor) { - if (constructor.isSynthetic()) { - return true; - } - Class[] parameterTypes = constructor.getParameterTypes(); - return parameterTypes.length > 0 - && "kotlin.jvm.internal.DefaultConstructorMarker" - .equals(parameterTypes[parameterTypes.length - 1].getName()); - } - - private static boolean isJavaPlatformType(Class type) { - return type.getName().startsWith("java."); - } - - private static Field[] fieldsByExplicitNames( - Class type, Constructor constructor, String[] names, String source) { - List fields = new ArrayList<>(); - fields.addAll(Descriptor.getFields(type)); - Map fieldsByName = new LinkedHashMap<>(); - Set duplicateNames = new LinkedHashSet<>(); - for (Field field : fields) { - Field previous = fieldsByName.put(field.getName(), field); - if (previous != null) { - duplicateNames.add(field.getName()); - } - } - Field[] constructorFields = new Field[names.length]; - Set seenNames = new LinkedHashSet<>(); - for (int i = 0; i < names.length; i++) { - String name = names[i]; - if (!seenNames.add(name)) { - throw new ForyException( - source + " constructor " + constructor + " maps " + name + " twice"); - } - if (duplicateNames.contains(name)) { - throw new ForyException( - source - + " constructor " - + constructor - + " cannot bind duplicate field name " - + name - + " in " - + type); - } - Field field = fieldsByName.get(name); - if (field == null) { - throw new ForyException( - source + " constructor " + constructor + " maps unknown field " + name); - } - constructorFields[i] = field; - } - return constructorFields; - } - - private static ConstructorMatch matchConstructorFields( - Constructor constructor, Field[] fields) { - Class[] parameterTypes = constructor.getParameterTypes(); - String[] names = new String[fields.length]; - Class[] declaringClasses = new Class[fields.length]; - Class[] fieldTypes = new Class[fields.length]; - boolean[] finalFieldFlags = new boolean[fields.length]; - for (int i = 0; i < fields.length; i++) { - Field field = fields[i]; - if (!constructorTypeMatches(parameterTypes[i], field)) { - return null; - } - names[i] = field.getName(); - declaringClasses[i] = field.getDeclaringClass(); - fieldTypes[i] = field.getType(); - finalFieldFlags[i] = Modifier.isFinal(field.getModifiers()); - } - return new ConstructorMatch<>( - constructor, names, declaringClasses, fieldTypes, finalFieldFlags); - } - - private static boolean constructorTypeMatches(Class parameterType, Field field) { - Class boxedParameterType = TypeUtils.boxedType(parameterType); - Class boxedFieldType = TypeUtils.boxedType(field.getType()); - return boxedParameterType.isAssignableFrom(boxedFieldType); - } - - private static MethodHandle constructorHandle(Class type, Constructor constructor) { - Lookup lookup = _JDKAccess._trustedLookup(type); - if (lookup == null) { - return null; - } - try { - MethodHandle handle = - lookup.findConstructor( - type, MethodType.methodType(void.class, constructor.getParameterTypes())); - return handle.asSpreader(Object[].class, constructor.getParameterCount()); - } catch (NoSuchMethodException | IllegalAccessException e) { - return null; - } - } - private static RuntimeException makeException(Class type, Throwable cause) { Throwable target = unwrapConstructorFailure(cause); // Keep constructor invocation failures outside ForyException so top-level deserialization can @@ -406,103 +190,6 @@ public T newInstanceWithArguments(Object... arguments) { } } - public static final class ConstructorObjectCreator extends ObjectCreator { - private final Constructor constructor; - private final MethodHandle handle; - private final String[] fieldNames; - private final Class[] declaringClasses; - private final Class[] fieldTypes; - private final boolean[] finalFields; - - private ConstructorObjectCreator(Class type, ConstructorMatch match) { - super(type); - constructor = match.constructor; - handle = constructorHandle(type, constructor); - fieldNames = match.fieldNames; - declaringClasses = match.declaringClasses; - fieldTypes = match.fieldTypes; - finalFields = match.finalFields; - if (handle == null) { - makeConstructorAccessible(type, constructor); - } - } - - @Override - public boolean hasConstructorFields() { - return true; - } - - @Override - public String[] getConstructorFieldNames() { - return fieldNames.clone(); - } - - @Override - public Class[] getConstructorFieldDeclaringClasses() { - return declaringClasses.clone(); - } - - @Override - public Class[] getConstructorFieldTypes() { - return fieldTypes.clone(); - } - - @Override - public boolean[] getConstructorFieldFinal() { - return finalFields.clone(); - } - - @Override - public boolean isConstructorPublic() { - return Modifier.isPublic(type.getModifiers()) - && Modifier.isPublic(constructor.getModifiers()); - } - - @Override - public boolean isOnlyPublicConstructor() { - if (!isConstructorPublic()) { - return false; - } - for (Constructor declaredConstructor : type.getDeclaredConstructors()) { - if (Modifier.isPublic(declaredConstructor.getModifiers()) - && declaredConstructor != constructor) { - return false; - } - } - return true; - } - - private static void makeConstructorAccessible(Class type, Constructor constructor) { - try { - constructor.setAccessible(true); - } catch (RuntimeException e) { - throw new ForyException("Failed to make constructor accessible for " + type, e); - } - } - - @Override - public T newInstance() { - throw constructorArgsRequired(type); - } - - @Override - public T newInstanceWithArguments(Object... arguments) { - try { - if (handle == null) { - return constructor.newInstance(arguments); - } - return (T) handle.invoke(arguments); - } catch (Throwable e) { - throw makeException(type, e); - } - } - - private static ForyException constructorArgsRequired(Class type) { - return new ForyException( - "JDK25 zero-Unsafe mode requires constructor field values to create " + type); - } - } - private static final class ConstructorBypassObjectCreator extends ObjectCreator { private final ConstructorBypassAllocator allocator; diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/SharedRegistry.java b/java/fory-core/src/main/java/org/apache/fory/resolver/SharedRegistry.java index a2a8ecebd5..c7311ea78b 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/SharedRegistry.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/SharedRegistry.java @@ -19,7 +19,6 @@ package org.apache.fory.resolver; -import java.lang.reflect.Constructor; import java.lang.reflect.Member; import java.util.ArrayList; import java.util.Collections; @@ -151,13 +150,6 @@ public ObjectCreator getObjectStreamCreator(Class type) { objectStreamCreatorCache.computeIfAbsent(type, ObjectCreators::createObjectStreamCreator); } - public void registerConstructor( - Class type, Constructor constructor, String... fieldNames) { - objectCreatorCache.put( - type, - ObjectCreators.createObjectCreator(type, constructor, fieldNames.clone(), "registered")); - } - TypeInfo cacheRegisteredTypeInfo(Class type, TypeInfo typeInfo) { TypeInfo existing = registeredTypeInfoCache.putIfAbsent(type, typeInfo); return existing == null ? typeInfo : existing; diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/AbstractObjectSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/AbstractObjectSerializer.java index 87a8984f4f..2ffac30bf1 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/AbstractObjectSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/AbstractObjectSerializer.java @@ -70,20 +70,15 @@ public abstract class AbstractObjectSerializer extends Serializer { private static final Logger LOG = LoggerFactory.getLogger(AbstractObjectSerializer.class); private static final Object SELF_REFERENCE = new Object(); // Constructor-bound objects reserve a ref id before constructor arguments are read, but the - // context package must stay a generic ref owner. Track unresolved constructor refs here so - // final-field construction does not add context APIs or bind the wrong pending ref id. + // object cannot be referenced semantically until the constructor returns. ReadContext calls the + // tracker from tryPreserveRefId so nested collection/map/array elements cannot hide an unresolved + // self-reference inside a constructor argument. private static final Object CONSTRUCTOR_REF_IDS = new Object(); private static final Object UNRESOLVED_CONSTRUCTOR_REF_IDS = new Object(); protected final Config config; protected final TypeResolver typeResolver; protected final boolean isRecord; protected final ObjectCreator objectCreator; - private final String[] objectCreatorConstructorFieldNames; - private final Class[] objectCreatorConstructorFieldDeclaringClasses; - private final Class[] objectCreatorConstructorFieldTypes; - private final boolean[] objectCreatorConstructorFieldFinal; - private volatile int[] copyConstructorFieldIndexes; - private volatile boolean[] copyConstructorFieldMask; private SerializationFieldInfo[] fieldInfos; private RecordInfo copyRecordInfo; @@ -93,10 +88,6 @@ protected AbstractObjectSerializer() { this.typeResolver = null; this.isRecord = false; this.objectCreator = null; - this.objectCreatorConstructorFieldNames = null; - this.objectCreatorConstructorFieldDeclaringClasses = null; - this.objectCreatorConstructorFieldTypes = null; - this.objectCreatorConstructorFieldFinal = null; } public AbstractObjectSerializer(TypeResolver typeResolver, Class type) { @@ -110,18 +101,6 @@ public AbstractObjectSerializer( this.typeResolver = typeResolver; this.isRecord = RecordUtils.isRecord(type); this.objectCreator = objectCreator; - if (objectCreator.hasConstructorFields()) { - this.objectCreatorConstructorFieldNames = objectCreator.getConstructorFieldNames(); - this.objectCreatorConstructorFieldDeclaringClasses = - objectCreator.getConstructorFieldDeclaringClasses(); - this.objectCreatorConstructorFieldTypes = objectCreator.getConstructorFieldTypes(); - this.objectCreatorConstructorFieldFinal = objectCreator.getConstructorFieldFinal(); - } else { - this.objectCreatorConstructorFieldNames = null; - this.objectCreatorConstructorFieldDeclaringClasses = null; - this.objectCreatorConstructorFieldTypes = null; - this.objectCreatorConstructorFieldFinal = null; - } } static void writeField( @@ -193,7 +172,6 @@ static Object readField( MemoryBuffer buffer) { if (fieldInfo.useDeclaredTypeInfo) { if (refMode == RefMode.TRACKING) { - trackConstructorRefRead(readContext, buffer); return readContext.readRef(fieldInfo.typeInfo); } if (refMode != RefMode.NULL_ONLY || buffer.readByte() != Fory.NULL_FLAG) { @@ -203,8 +181,7 @@ static Object readField( return null; } if (refMode == RefMode.TRACKING) { - trackConstructorRefRead(readContext, buffer); - int nextReadRefId = refReader.tryPreserveRefId(buffer); + int nextReadRefId = readContext.tryPreserveRefId(); if (nextReadRefId >= Fory.NOT_NULL_VALUE_FLAG) { Object value = typeResolver @@ -530,7 +507,7 @@ static Object readContainerFieldValue( case TRACKING: generics.pushGenericType(fieldInfo.genericType, readContext.getDepth()); fieldValue = - readContainerFieldValueRef(readContext, typeResolver, refReader, fieldInfo, buffer); + readContainerFieldValueRef(readContext, typeResolver, refReader, fieldInfo); generics.popGenericType(readContext.getDepth()); break; default: @@ -554,10 +531,8 @@ private static Object readContainerFieldValueRef( ReadContext readContext, TypeResolver typeResolver, RefReader refReader, - SerializationFieldInfo fieldInfo, - MemoryBuffer buffer) { - trackConstructorRefRead(readContext, buffer); - int nextReadRefId = refReader.tryPreserveRefId(buffer); + SerializationFieldInfo fieldInfo) { + int nextReadRefId = readContext.tryPreserveRefId(); if (nextReadRefId >= Fory.NOT_NULL_VALUE_FLAG) { Object value; if (fieldInfo.containerSerializerOverride != null) { @@ -900,9 +875,6 @@ public T copy(CopyContext copyContext, T originObj) { if (isRecord) { return copyRecord(copyContext, originObj); } - if (objectCreator.hasConstructorFields()) { - return copyConstructorObject(copyContext, originObj); - } T newObj = newBean(); copyContext.reference(originObj, newObj); copyFields(copyContext, originObj, newObj); @@ -923,45 +895,6 @@ private T copyRecord(CopyContext copyContext, T originObj) { return originObj; } - private T copyConstructorObject(CopyContext copyContext, T originObj) { - SerializationFieldInfo[] fieldInfos = this.fieldInfos; - if (fieldInfos == null) { - fieldInfos = buildFieldsInfo(); - } - int[] constructorFieldIndexes = copyConstructorFieldIndexes(fieldInfos); - boolean[] constructorFieldMask = copyConstructorFieldMask(fieldInfos, constructorFieldIndexes); - Object pendingMarker = beginConstructorCopy(copyContext, originObj); - Object[] fieldValues = - copyFieldValues(copyContext, originObj, fieldInfos, constructorFieldMask, true); - checkNoConstructorCopyBackrefs(fieldValues, constructorFieldIndexes, pendingMarker); - T newObj = - objectCreator.newInstanceWithArguments( - constructorArgs( - fieldValues, constructorFieldIndexes, objectCreatorConstructorFieldTypes)); - copyContext.reference(originObj, newObj); - copyFields(copyContext, fieldInfos, originObj, newObj, constructorFieldMask, false); - return newObj; - } - - private int[] copyConstructorFieldIndexes(SerializationFieldInfo[] fieldInfos) { - int[] indexes = copyConstructorFieldIndexes; - if (indexes == null) { - indexes = buildConstructorFieldIndexes(fieldInfos); - copyConstructorFieldIndexes = indexes; - } - return indexes; - } - - private boolean[] copyConstructorFieldMask( - SerializationFieldInfo[] fieldInfos, int[] constructorFieldIndexes) { - boolean[] mask = copyConstructorFieldMask; - if (mask == null) { - mask = buildConstructorFieldMask(fieldInfos.length, constructorFieldIndexes); - copyConstructorFieldMask = mask; - } - return mask; - } - private Object[] copyFieldValues(CopyContext copyContext, T originObj) { SerializationFieldInfo[] fieldInfos = this.fieldInfos; if (fieldInfos == null) { @@ -981,29 +914,6 @@ private Object[] copyFieldValues(CopyContext copyContext, T originObj) { return fieldValues; } - private Object[] copyFieldValues( - CopyContext copyContext, - T originObj, - SerializationFieldInfo[] fieldInfos, - boolean[] constructorFieldMask, - boolean constructorFields) { - Object[] fieldValues = new Object[fieldInfos.length]; - for (int i = 0; i < fieldInfos.length; i++) { - if (constructorFieldMask[i] != constructorFields) { - continue; - } - SerializationFieldInfo fieldInfo = fieldInfos[i]; - FieldAccessor fieldAccessor = fieldInfo.fieldAccessor; - if (fieldInfo.isPrimitiveField) { - fieldValues[i] = copyPrimitiveField(originObj, fieldAccessor, fieldInfo.dispatchId); - } else { - fieldValues[i] = - copyNotPrimitiveField(copyContext, originObj, fieldAccessor, fieldInfo.dispatchId); - } - } - return fieldValues; - } - private void copyFields(CopyContext copyContext, T originObj, T newObj) { SerializationFieldInfo[] fieldInfos = this.fieldInfos; if (fieldInfos == null) { @@ -1028,28 +938,6 @@ public static void copyFields( } } - private static void copyFields( - CopyContext copyContext, - SerializationFieldInfo[] fieldInfos, - Object originObj, - Object newObj, - boolean[] constructorFieldMask, - boolean constructorFields) { - for (int i = 0; i < fieldInfos.length; i++) { - if (constructorFieldMask[i] != constructorFields) { - continue; - } - SerializationFieldInfo fieldInfo = fieldInfos[i]; - FieldAccessor fieldAccessor = fieldInfo.fieldAccessor; - if (fieldInfo.isPrimitiveField) { - copySetPrimitiveField(originObj, newObj, fieldAccessor, fieldInfo.dispatchId); - } else { - copySetNotPrimitiveField( - copyContext, originObj, newObj, fieldAccessor, fieldInfo.dispatchId); - } - } - } - private static Object copyFieldValue(CopyContext copyContext, Object fieldValue, int dispatchId) { if (fieldValue == null) { return null; @@ -1199,106 +1087,6 @@ protected T newBean() { return objectCreator.newInstance(); } - protected final String[] constructorFieldNames() { - return objectCreatorConstructorFieldNames; - } - - protected final Class[] constructorFieldDeclaringClasses() { - return objectCreatorConstructorFieldDeclaringClasses; - } - - protected final Class[] constructorFieldTypes() { - return objectCreatorConstructorFieldTypes; - } - - protected final boolean[] constructorFieldFinal() { - return objectCreatorConstructorFieldFinal; - } - - protected final int[] buildConstructorFieldIndexes(SerializationFieldInfo[] fieldInfos) { - return buildConstructorFieldIndexes(fieldInfos, true); - } - - protected final int[] buildConstructorFieldIndexes( - SerializationFieldInfo[] fieldInfos, boolean allowMissingNonFinal) { - return buildConstructorFieldIndexes(fieldInfos, allowMissingNonFinal, null); - } - - protected final int[] buildConstructorFieldIndexes( - SerializationFieldInfo[] fieldInfos, boolean allowMissingNonFinal, String[] defaultFields) { - return buildConstructorFieldIndexes(fieldInfos, allowMissingNonFinal, defaultFields, null); - } - - protected final int[] buildConstructorFieldIndexes( - SerializationFieldInfo[] fieldInfos, - boolean allowMissingNonFinal, - String[] defaultFields, - Class[] defaultDeclaringClasses) { - String[] fieldNames = objectCreatorConstructorFieldNames; - if (fieldNames.length == 0) { - return null; - } - Class[] declaringClasses = objectCreatorConstructorFieldDeclaringClasses; - boolean[] finalFields = objectCreatorConstructorFieldFinal; - int[] indexes = new int[fieldNames.length]; - for (int i = 0; i < fieldNames.length; i++) { - Class declaringClass = declaringClasses == null ? null : declaringClasses[i]; - boolean allowMissing = - (allowMissingNonFinal && !finalFields[i]) - || contains(defaultFields, defaultDeclaringClasses, fieldNames[i], declaringClass); - indexes[i] = constructorFieldIndex(fieldInfos, declaringClass, fieldNames[i], allowMissing); - } - return indexes; - } - - private static boolean contains( - String[] values, Class[] declaringClasses, String value, Class declaringClass) { - if (values == null) { - return false; - } - for (int i = 0; i < values.length; i++) { - if (values[i].equals(value) - && (declaringClasses == null - || i >= declaringClasses.length - || declaringClasses[i] == null - || declaringClasses[i] == declaringClass)) { - return true; - } - } - return false; - } - - protected final boolean[] buildConstructorFieldMask(int size, int[] indexes) { - if (indexes == null) { - return null; - } - boolean[] mask = new boolean[size]; - for (int index : indexes) { - if (index >= 0) { - mask[index] = true; - } - } - return mask; - } - - protected final Object[] constructorArgs(Object[] fieldValues, int[] indexes) { - Object[] args = new Object[indexes.length]; - for (int i = 0; i < indexes.length; i++) { - args[i] = fieldValues[indexes[i]]; - } - return args; - } - - protected final Object[] constructorArgs( - Object[] fieldValues, int[] indexes, Class[] fieldTypes) { - Object[] args = new Object[indexes.length]; - for (int i = 0; i < indexes.length; i++) { - int index = indexes[i]; - args[i] = index < 0 ? defaultConstructorValue(fieldTypes[i]) : fieldValues[index]; - } - return args; - } - protected final void checkNoUnresolvedReadRef(ReadContext readContext) { checkNoUnresolvedReadRef(readContext, type); } @@ -1551,77 +1339,9 @@ private static boolean consumeUnresolvedConstructorRef(ReadContext readContext, protected static void throwConstructorCycle(Class type) { throw new ForyException( - "Cyclic references to constructor-bound type " + "Cyclic references to constructor-created type " + type.getName() + " cannot be restored before the object is constructed. Use a no-arg constructor " - + "or keep the cycle as a direct non-constructor field."); - } - - public static Object defaultConstructorValue(Class type) { - if (type == boolean.class) { - return false; - } else if (type == byte.class) { - return (byte) 0; - } else if (type == short.class) { - return (short) 0; - } else if (type == char.class) { - return (char) 0; - } else if (type == int.class) { - return 0; - } else if (type == long.class) { - return 0L; - } else if (type == float.class) { - return 0.0f; - } else if (type == double.class) { - return 0.0d; - } - return null; - } - - protected final void setNonConstructorFields( - Object targetObject, - Object[] fieldValues, - SerializationFieldInfo[] fieldInfos, - boolean[] constructorMask) { - for (int i = 0; i < fieldInfos.length; i++) { - if (!constructorMask[i] && fieldInfos[i].fieldAccessor != null) { - fieldInfos[i].fieldAccessor.putObject(targetObject, fieldValues[i]); - } - } - } - - public static Object fieldValue(Object[] fieldValues, int index) { - return fieldValues[index]; - } - - private static int constructorFieldIndex( - SerializationFieldInfo[] fieldInfos, - Class declaringClass, - String fieldName, - boolean allowMissing) { - int index = -1; - for (int i = 0; i < fieldInfos.length; i++) { - FieldAccessor fieldAccessor = fieldInfos[i].fieldAccessor; - if (fieldAccessor == null) { - continue; - } - Field field = fieldAccessor.getField(); - if (!field.getName().equals(fieldName) - || (declaringClass != null && field.getDeclaringClass() != declaringClass)) { - continue; - } - if (index >= 0) { - throw new ForyException( - "Constructor field " + fieldName + " is ambiguous because multiple fields match"); - } - index = i; - } - if (index < 0) { - if (allowMissing) { - return -1; - } - throw new ForyException("Constructor field " + fieldName + " is not serialized"); - } - return index; + + "or keep the cycle outside constructor parameters."); } } diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/CompatibleCollectionArrayReader.java b/java/fory-core/src/main/java/org/apache/fory/serializer/CompatibleCollectionArrayReader.java index ad8e22ef62..42b1436e90 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/CompatibleCollectionArrayReader.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/CompatibleCollectionArrayReader.java @@ -307,9 +307,7 @@ private static Object readTracking( int elementTypeId, Class targetType) { RefReader refReader = readContext.getRefReader(); - MemoryBuffer buffer = readContext.getBuffer(); - AbstractObjectSerializer.trackConstructorRefRead(readContext, buffer); - int nextReadRefId = refReader.tryPreserveRefId(buffer); + int nextReadRefId = readContext.tryPreserveRefId(); if (nextReadRefId >= Fory.NOT_NULL_VALUE_FLAG) { Object value = readNotNull(readContext, readMode, arrayTypeId, elementTypeId, targetType); refReader.setReadRef(nextReadRefId, value); diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/CompatibleSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/CompatibleSerializer.java index f734b6980d..ca88cebde3 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/CompatibleSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/CompatibleSerializer.java @@ -31,9 +31,6 @@ import org.apache.fory.logging.LoggerFactory; import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.meta.TypeDef; -import org.apache.fory.platform.AndroidSupport; -import org.apache.fory.platform.GraalvmSupport; -import org.apache.fory.platform.JdkVersion; import org.apache.fory.reflect.FieldAccessor; import org.apache.fory.resolver.ClassResolver; import org.apache.fory.resolver.RefMode; @@ -68,11 +65,6 @@ public class CompatibleSerializer extends AbstractObjectSerializer { private static final Logger LOG = LoggerFactory.getLogger(CompatibleSerializer.class); private final SerializationFieldInfo[] allFields; - private final int[] constructorFieldIndexes; - private final boolean[] constructorFieldMask; - private final String[] constructorFieldNames; - private final Class[] constructorFieldDeclaringClasses; - private final Class[] constructorFieldTypes; private final CompatibleCollectionArrayReader.ReadAction[] allCompatibleReadActions; private final boolean hasCompatibleCollectionArrayRead; private final RecordInfo recordInfo; @@ -144,40 +136,6 @@ public CompatibleSerializer(TypeResolver typeResolver, Class type, TypeDef ty } this.hasDefaultValues = hasDefaultValues; this.defaultValueFields = defaultValueFields; - if (!isRecord && objectCreator.hasConstructorFields()) { - constructorFieldIndexes = - buildConstructorFieldIndexes( - allFields, - true, - defaultFieldNames(defaultValueFields), - defaultDeclaringClasses(defaultValueFields)); - constructorFieldMask = buildConstructorFieldMask(allFields.length, constructorFieldIndexes); - constructorFieldNames = constructorFieldNames(); - constructorFieldDeclaringClasses = constructorFieldDeclaringClasses(); - constructorFieldTypes = constructorFieldTypes(); - } else { - constructorFieldIndexes = null; - constructorFieldMask = null; - constructorFieldNames = null; - constructorFieldDeclaringClasses = null; - constructorFieldTypes = null; - } - } - - private static String[] defaultFieldNames(DefaultValueUtils.DefaultValueField[] fields) { - String[] names = new String[fields.length]; - for (int i = 0; i < fields.length; i++) { - names[i] = fields[i].getFieldName(); - } - return names; - } - - private static Class[] defaultDeclaringClasses(DefaultValueUtils.DefaultValueField[] fields) { - Class[] declaringClasses = new Class[fields.length]; - for (int i = 0; i < fields.length; i++) { - declaringClasses[i] = fields[i].getDeclaringClass(); - } - return declaringClasses; } /** Used by generated compatible serializers for top-level list/array compatible field reads. */ @@ -270,45 +228,12 @@ private T newInstance() { if (!hasDefaultValues) { return newBean(); } - T obj = - AndroidSupport.IS_ANDROID - || GraalvmSupport.IN_GRAALVM_NATIVE_IMAGE - || JdkVersion.MAJOR_VERSION >= 25 - ? newBean() - : typeResolver.getObjectCreator(type).newInstance(); + T obj = newBean(); // Set default values for missing fields in Scala case classes DefaultValueUtils.setDefaultValues(obj, defaultValueFields); return obj; } - private Object[] compatibleConstructorArgs(Object[] fieldValues) { - Object[] args = new Object[constructorFieldIndexes.length]; - for (int i = 0; i < constructorFieldIndexes.length; i++) { - int index = constructorFieldIndexes[i]; - if (index >= 0) { - args[i] = fieldValues[index]; - } else { - Class declaringClass = - constructorFieldDeclaringClasses == null ? null : constructorFieldDeclaringClasses[i]; - args[i] = - defaultConstructorValue( - constructorFieldNames[i], declaringClass, constructorFieldTypes[i]); - } - } - return args; - } - - private Object defaultConstructorValue( - String fieldName, Class declaringClass, Class fieldType) { - for (DefaultValueUtils.DefaultValueField defaultValueField : defaultValueFields) { - if (defaultValueField.getFieldName().equals(fieldName) - && (declaringClass == null || defaultValueField.getDeclaringClass() == declaringClass)) { - return defaultValueField.getDefaultValue(); - } - } - return AbstractObjectSerializer.defaultConstructorValue(fieldType); - } - @Override public T read(ReadContext readContext) { if (isRecord) { @@ -323,10 +248,6 @@ public T read(ReadContext readContext) { Arrays.fill(recordInfo.getRecordComponents(), null); return t; } - if (objectCreator.hasConstructorFields()) { - Object[] fieldValues = new Object[allFields.length]; - return readConstructorObject(readContext, fieldValues); - } T targetObject = newInstance(); if (readContext.hasPreservedRefId()) { readContext.reference(targetObject); @@ -339,85 +260,6 @@ public T read(ReadContext readContext) { return targetObject; } - private T readConstructorObject(ReadContext readContext, Object[] fieldValues) { - beginConstructorRef(readContext); - try { - boolean[] bufferedNonConstructorFields = new boolean[allFields.length]; - int remainingConstructorFields = countConstructorFields(); - T targetObject = null; - if (remainingConstructorFields == 0) { - targetObject = createConstructorObject(fieldValues); - referenceConstructorRef(readContext, targetObject); - setNonConstructorDefaultValues(targetObject); - } - MemoryBuffer buffer = readContext.getBuffer(); - RefReader refReader = readContext.getRefReader(); - Generics generics = readContext.getGenerics(); - for (int i = 0; i < allFields.length; i++) { - SerializationFieldInfo fieldInfo = allFields[i]; - CompatibleCollectionArrayReader.ReadAction action = - compatibleCollectionArrayReadAction(allCompatibleReadActions, i); - if (constructorFieldMask[i]) { - fieldValues[i] = - ctorFieldValue( - readContext, - readFieldValue(readContext, refReader, generics, fieldInfo, buffer, action), - type); - remainingConstructorFields--; - if (remainingConstructorFields == 0) { - checkNoUnresolvedReadRef(readContext); - targetObject = createConstructorObject(fieldValues); - referenceConstructorRef(readContext, targetObject); - setNonConstructorDefaultValues(targetObject); - setBufferedNonConstructorFields( - targetObject, fieldValues, bufferedNonConstructorFields); - } - } else if (targetObject == null) { - fieldValues[i] = - bufferFieldValue( - readContext, - readFieldValue(readContext, refReader, generics, fieldInfo, buffer, action), - type); - bufferedNonConstructorFields[i] = true; - } else { - readField(readContext, targetObject, refReader, generics, fieldInfo, buffer, action); - } - } - return targetObject; - } finally { - endConstructorRef(readContext); - } - } - - private int countConstructorFields() { - int count = 0; - for (boolean constructorField : constructorFieldMask) { - if (constructorField) { - count++; - } - } - return count; - } - - private T createConstructorObject(Object[] fieldValues) { - return objectCreator.newInstanceWithArguments(compatibleConstructorArgs(fieldValues)); - } - - private void setNonConstructorDefaultValues(T targetObject) { - DefaultValueUtils.setDefaultValues( - targetObject, defaultValueFields, constructorFieldNames, constructorFieldDeclaringClasses); - } - - private void setBufferedNonConstructorFields( - T targetObject, Object[] fieldValues, boolean[] bufferedNonConstructorFields) { - for (int i = 0; i < allFields.length; i++) { - if (bufferedNonConstructorFields[i]) { - setFieldValue( - targetObject, allFields[i], resolveBufferedValue(fieldValues[i], targetObject)); - } - } - } - private void setFieldValue(T targetObject, SerializationFieldInfo fieldInfo, Object fieldValue) { if (fieldInfo.fieldAccessor != null) { fieldInfo.fieldAccessor.putObject(targetObject, fieldValue); diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectSerializer.java index 74e45e0029..1ef5f1fbfb 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectSerializer.java @@ -64,9 +64,6 @@ public final class ObjectSerializer extends AbstractObjectSerializer { private final RecordInfo recordInfo; private final SerializationFieldInfo[] allFields; - private final int[] constructorFieldIndexes; - private final boolean[] constructorFieldMask; - private final Class[] constructorFieldTypes; private final int classVersionHash; public ObjectSerializer(TypeResolver typeResolver, Class cls) { @@ -138,15 +135,6 @@ public ObjectSerializer( } FieldGroups fieldGroups = FieldGroups.buildFieldInfos(typeResolver, grouper); allFields = fieldGroups.allFields; - if (!isRecord && objectCreator.hasConstructorFields()) { - constructorFieldIndexes = buildConstructorFieldIndexes(allFields); - constructorFieldMask = buildConstructorFieldMask(allFields.length, constructorFieldIndexes); - constructorFieldTypes = constructorFieldTypes(); - } else { - constructorFieldIndexes = null; - constructorFieldMask = null; - constructorFieldTypes = null; - } } @Override @@ -228,88 +216,11 @@ public T read(ReadContext readContext) { Arrays.fill(recordInfo.getRecordComponents(), null); return obj; } - if (objectCreator.hasConstructorFields()) { - return readConstructorObject(readContext); - } T obj = newBean(); readContext.reference(obj); return readAndSetFields(readContext, obj); } - private T readConstructorObject(ReadContext readContext) { - beginConstructorRef(readContext); - try { - MemoryBuffer buffer = readContext.getBuffer(); - if (typeResolver.checkClassVersion()) { - int hash = buffer.readInt32(); - checkClassVersion(type, hash, classVersionHash); - } - Object[] fieldValues = new Object[allFields.length]; - boolean[] bufferedNonConstructorFields = new boolean[allFields.length]; - int remainingConstructorFields = countConstructorFields(); - T obj = null; - if (remainingConstructorFields == 0) { - obj = createConstructorObject(fieldValues); - referenceConstructorRef(readContext, obj); - } - RefReader refReader = readContext.getRefReader(); - Generics generics = readContext.getGenerics(); - for (int i = 0; i < allFields.length; i++) { - SerializationFieldInfo fieldInfo = allFields[i]; - if (constructorFieldMask[i]) { - fieldValues[i] = - ctorFieldValue( - readContext, - readFieldByCodecCategory(readContext, refReader, generics, fieldInfo, buffer), - type); - remainingConstructorFields--; - if (remainingConstructorFields == 0) { - checkNoUnresolvedReadRef(readContext); - obj = createConstructorObject(fieldValues); - referenceConstructorRef(readContext, obj); - setBufferedNonConstructorFields(obj, fieldValues, bufferedNonConstructorFields); - } - } else if (obj == null) { - fieldValues[i] = - bufferFieldValue( - readContext, - readFieldByCodecCategory(readContext, refReader, generics, fieldInfo, buffer), - type); - bufferedNonConstructorFields[i] = true; - } else { - readAndSetFieldByCodecCategory(readContext, refReader, generics, fieldInfo, buffer, obj); - } - } - return obj; - } finally { - endConstructorRef(readContext); - } - } - - private int countConstructorFields() { - int count = 0; - for (boolean constructorField : constructorFieldMask) { - if (constructorField) { - count++; - } - } - return count; - } - - private T createConstructorObject(Object[] fieldValues) { - return objectCreator.newInstanceWithArguments( - constructorArgs(fieldValues, constructorFieldIndexes, constructorFieldTypes)); - } - - private void setBufferedNonConstructorFields( - T obj, Object[] fieldValues, boolean[] bufferedNonConstructorFields) { - for (int i = 0; i < allFields.length; i++) { - if (bufferedNonConstructorFields[i]) { - allFields[i].fieldAccessor.putObject(obj, resolveBufferedValue(fieldValues[i], obj)); - } - } - } - public Object[] readFields(ReadContext readContext) { MemoryBuffer buffer = readContext.getBuffer(); RefReader refReader = readContext.getRefReader(); diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/StaticGeneratedStructSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/StaticGeneratedStructSerializer.java index a13c7934e1..886cd613f8 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/StaticGeneratedStructSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/StaticGeneratedStructSerializer.java @@ -248,47 +248,6 @@ public final int[] localFieldIds( return ids; } - protected final int[] buildConstructorFieldIds(List descriptors) { - String[] fieldNames = constructorFieldNames(); - if (fieldNames.length == 0) { - return null; - } - Class[] declaringClasses = constructorFieldDeclaringClasses(); - boolean[] finalFields = constructorFieldFinal(); - int[] ids = new int[fieldNames.length]; - for (int i = 0; i < fieldNames.length; i++) { - Class declaringClass = declaringClasses == null ? null : declaringClasses[i]; - ids[i] = constructorFieldId(descriptors, declaringClass, fieldNames[i], !finalFields[i]); - } - return ids; - } - - private int constructorFieldId( - List descriptors, - Class declaringClass, - String fieldName, - boolean allowMissing) { - int id = UNKNOWN_FIELD; - String declaringClassName = declaringClass == null ? null : declaringClass.getName(); - for (int i = 0; i < descriptors.size(); i++) { - Descriptor descriptor = descriptors.get(i); - if (!descriptor.getName().equals(fieldName) - || (declaringClassName != null - && !descriptor.getDeclaringClass().equals(declaringClassName))) { - continue; - } - if (id != UNKNOWN_FIELD) { - throw new ForyException( - "Constructor field " + fieldName + " is ambiguous because multiple fields match"); - } - id = i; - } - if (id == UNKNOWN_FIELD && !allowMissing) { - throw new ForyException("Constructor field " + fieldName + " is not serialized"); - } - return id; - } - protected final long[] buildConstructorFieldBits(int size, int[] indexes) { if (indexes == null) { return null; diff --git a/java/fory-core/src/main/java25/org/apache/fory/reflect/ConstructorBypassAllocator.java b/java/fory-core/src/main/java25/org/apache/fory/reflect/ConstructorBypassAllocator.java index 8fbd5a44c9..0a18869817 100644 --- a/java/fory-core/src/main/java25/org/apache/fory/reflect/ConstructorBypassAllocator.java +++ b/java/fory-core/src/main/java25/org/apache/fory/reflect/ConstructorBypassAllocator.java @@ -57,9 +57,7 @@ private static ForyException unsupported(Class type, Throwable cause) { "Cannot create a constructor-bypassing instance for " + type + " in JDK25+ zero-Unsafe mode. Provide an accessible no-arg constructor, " - + "annotate a constructor with @ForyConstructor, register a constructor with " - + "BaseFory.registerConstructor, use a record canonical constructor, or register a " - + "custom serializer.", + + "use a record canonical constructor, or register a custom serializer.", cause); } diff --git a/java/fory-core/src/main/java25/org/apache/fory/serializer/PlatformStringUtils.java b/java/fory-core/src/main/java25/org/apache/fory/serializer/PlatformStringUtils.java index 81d55e98c6..5f347dad39 100644 --- a/java/fory-core/src/main/java25/org/apache/fory/serializer/PlatformStringUtils.java +++ b/java/fory-core/src/main/java25/org/apache/fory/serializer/PlatformStringUtils.java @@ -89,8 +89,11 @@ private static StringHandles stringHandles() { offsetField == null ? null : stringLookup.findVarHandle(String.class, "offset", int.class)); - } catch (Throwable ignored) { - return StringHandles.noAccess(); + } catch (Throwable e) { + throw new IllegalStateException( + "JDK25+ string internals require java.base/java.lang.invoke to be open to " + + "org.apache.fory.core", + e); } } catch (NoSuchFieldException e) { throw new RuntimeException(e); diff --git a/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties b/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties index 498c002bab..559b47c218 100644 --- a/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties +++ b/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties @@ -259,7 +259,6 @@ Args=--initialize-at-build-time=org.apache.fory.annotation.ForyField$Dynamic,\ org.apache.fory.reflect.TypeRef$TypeVariableKey,\ org.apache.fory.reflect.TypeRef,\ org.apache.fory.reflect.ObjectCreators,\ - org.apache.fory.reflect.ObjectCreators$ConstructorObjectCreator,\ org.apache.fory.reflect.ObjectCreators$ConstructorBypassObjectCreator,\ org.apache.fory.reflect.ObjectCreators$DeclaredNoArgCtrObjectCreator,\ org.apache.fory.reflect.ObjectCreators$ParentNoArgCtrObjectCreator,\ diff --git a/java/fory-core/src/test/java/org/apache/fory/ForyTest.java b/java/fory-core/src/test/java/org/apache/fory/ForyTest.java index 11b3671203..1832730930 100644 --- a/java/fory-core/src/test/java/org/apache/fory/ForyTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/ForyTest.java @@ -53,7 +53,6 @@ import lombok.Data; import lombok.EqualsAndHashCode; import org.apache.fory.annotation.Expose; -import org.apache.fory.annotation.ForyConstructor; import org.apache.fory.annotation.Ignore; import org.apache.fory.builder.Generated; import org.apache.fory.config.ForyBuilder; @@ -458,7 +457,6 @@ private static class IgnoreFields { this.f3 = f3; } - @ForyConstructor({"f3"}) IgnoreFields(long f3) { this.f3 = f3; } @@ -494,7 +492,6 @@ private static class ExposeFields { this.map2 = map2; } - @ForyConstructor({"f1", "f2", "map1"}) ExposeFields(int f1, long f2, ImmutableMap map1) { this.f1 = f1; this.f2 = f2; @@ -521,7 +518,6 @@ private static class ExposeFields2 { @Ignore long f2; long f3; - @ForyConstructor({"f1", "f2", "f3"}) ExposeFields2(int f1, long f2, long f3) { this.f1 = f1; this.f2 = f2; @@ -722,7 +718,6 @@ static class Struct1 { int f1; String f2; - @ForyConstructor({"f1", "f2"}) public Struct1(int f1, String f2) { this.f1 = f1; this.f2 = f2; @@ -785,7 +780,6 @@ static class MaxDepth { int f1; Object f2; - @ForyConstructor({"f1", "f2"}) MaxDepth(int f1, Object f2) { this.f1 = f1; this.f2 = f2; diff --git a/java/fory-core/src/test/java/org/apache/fory/ThreadSafeForyTest.java b/java/fory-core/src/test/java/org/apache/fory/ThreadSafeForyTest.java index 42aeff6634..19bcf03008 100644 --- a/java/fory-core/src/test/java/org/apache/fory/ThreadSafeForyTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/ThreadSafeForyTest.java @@ -25,7 +25,6 @@ import static org.testng.Assert.assertSame; import static org.testng.Assert.assertTrue; -import java.lang.reflect.Constructor; import java.nio.ByteBuffer; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CountDownLatch; @@ -99,71 +98,6 @@ public void testFactoryConstructorsClassLoader() { assertSame(threadPool.execute(Fory::getClassLoader), custom); } - @Test - public void testThreadLocalCtorRegFailure() throws Exception { - ThreadLocalFory fory = - Fory.builder() - .withXlang(false) - .withCodegen(false) - .requireClassRegistration(false) - .buildThreadLocalFory(); - fory.execute( - f -> { - f.getSerializer(ThreadSafeCtorBean.class); - return null; - }); - Constructor constructor = - ThreadSafeCtorBean.class.getDeclaredConstructor(int.class, String.class); - Assert.assertThrows( - ForyException.class, - () -> fory.registerConstructor(ThreadSafeCtorBean.class, constructor, "age", "name")); - assertFutureThreadCreator(fory, false); - } - - @Test - public void testThreadLocalCtorRegCopies() throws Exception { - ThreadLocalFory fory = - Fory.builder() - .withXlang(false) - .withCodegen(false) - .requireClassRegistration(false) - .buildThreadLocalFory(); - Constructor constructor = - ThreadSafeCtorBean.class.getDeclaredConstructor(int.class, String.class); - String[] fieldNames = {"age", "name"}; - fory.registerConstructor(ThreadSafeCtorBean.class, constructor, fieldNames); - fieldNames[0] = "name"; - fieldNames[1] = "age"; - assertFutureThreadCreator(fory, true); - } - - @Test - public void testThreadPoolCtorRegPreflight() throws Exception { - ThreadSafeFory fory = - Fory.builder() - .withXlang(false) - .withCodegen(false) - .requireClassRegistration(false) - .buildThreadSafeForyPool(2); - fory.execute( - f -> { - f.getSerializer(ThreadSafeCtorBean.class); - return null; - }); - Constructor constructor = - ThreadSafeCtorBean.class.getDeclaredConstructor(int.class, String.class); - Assert.assertThrows( - ForyException.class, - () -> fory.registerConstructor(ThreadSafeCtorBean.class, constructor, "age", "name")); - assertEquals( - fory.execute( - f -> - f.getTypeResolver() - .getObjectCreator(ThreadSafeCtorBean.class) - .hasConstructorFields()), - Boolean.FALSE); - } - @Test public void testThreadSafeRuntimesShareRegistry() throws Exception { ThreadLocalFory threadLocal = @@ -489,48 +423,12 @@ private static ByteBuffer[] byteBufferViews(byte[] payload) { return new ByteBuffer[] {heap, heapReadOnly, direct, directReadOnly}; } - private static void assertFutureThreadCreator(ThreadSafeFory fory, boolean constructorFields) - throws Exception { - AtomicReference result = new AtomicReference<>(); - AtomicReference error = new AtomicReference<>(); - Thread thread = - new Thread( - () -> { - try { - result.set( - fory.execute( - f -> - f.getTypeResolver() - .getObjectCreator(ThreadSafeCtorBean.class) - .hasConstructorFields())); - } catch (Throwable t) { - error.set(t); - } - }); - thread.start(); - thread.join(); - if (error.get() != null) { - throw new AssertionError(error.get()); - } - assertEquals(result.get(), Boolean.valueOf(constructorFields)); - } - private static byte[] wrapWithPadding(byte[] payload) { byte[] bytes = new byte[payload.length + 6]; System.arraycopy(payload, 0, bytes, 3, payload.length); return bytes; } - public static final class ThreadSafeCtorBean { - final int age; - final String name; - - private ThreadSafeCtorBean(int age, String name) { - this.age = age; - this.name = name; - } - } - @Data static class Foo { int f1; diff --git a/java/fory-core/src/test/java/org/apache/fory/codegen/pkgprivate/PackagePrivateMapKeyTest.java b/java/fory-core/src/test/java/org/apache/fory/codegen/pkgprivate/PackagePrivateMapKeyTest.java index 0f9076cf06..4124695b6b 100644 --- a/java/fory-core/src/test/java/org/apache/fory/codegen/pkgprivate/PackagePrivateMapKeyTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/codegen/pkgprivate/PackagePrivateMapKeyTest.java @@ -29,7 +29,6 @@ import java.util.Set; import org.apache.fory.Fory; import org.apache.fory.ThreadSafeFory; -import org.apache.fory.annotation.ForyConstructor; import org.testng.annotations.Test; /** Regression test for codegen CompileException when map key/value types are package-private. */ @@ -70,7 +69,6 @@ class ReproNode implements Serializable { Set children; Map> parents; - @ForyConstructor({"type", "id"}) ReproNode(ReproType type, String id) { this(type, id, new HashSet<>(), new EnumMap<>(ReproType.class)); } @@ -92,7 +90,6 @@ class ReproContainer implements Serializable { this(new EnumMap<>(ReproType.class), version); } - @ForyConstructor({"nodes", "version"}) ReproContainer(Map> nodes, String version) { this.nodes = nodes; this.version = version; diff --git a/java/fory-core/src/test/java/org/apache/fory/reflect/ObjectCreatorsTest.java b/java/fory-core/src/test/java/org/apache/fory/reflect/ObjectCreatorsTest.java index 118bbad57b..bda955b4c4 100644 --- a/java/fory-core/src/test/java/org/apache/fory/reflect/ObjectCreatorsTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/reflect/ObjectCreatorsTest.java @@ -22,7 +22,6 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; -import java.lang.reflect.Constructor; import java.nio.charset.StandardCharsets; import java.util.concurrent.ArrayBlockingQueue; import org.apache.fory.TestUtils; @@ -30,7 +29,6 @@ import org.apache.fory.platform.AndroidSupport; import org.apache.fory.platform.JdkVersion; import org.apache.fory.reflect.ObjectCreators.ParentNoArgCtrObjectCreator; -import org.apache.fory.resolver.SharedRegistry; import org.testng.Assert; import org.testng.annotations.Test; @@ -68,19 +66,6 @@ public void testAndroidObjectCreators() throws Exception { Assert.assertEquals(process.waitFor(), 0, output); } - @Test - public void testFailedCtorRegNotPublished() throws Exception { - if (JdkVersion.MAJOR_VERSION < 9) { - return; - } - SharedRegistry registry = new SharedRegistry(); - Constructor constructor = String.class.getDeclaredConstructor(byte[].class, byte.class); - Assert.assertThrows( - ForyException.class, - () -> registry.registerConstructor(String.class, constructor, "value", "coder")); - Assert.assertFalse(registry.getObjectCreator(String.class).hasConstructorFields()); - } - private static String readFully(InputStream inputStream) throws IOException { ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); byte[] buffer = new byte[1024]; diff --git a/java/fory-core/src/test/java/org/apache/fory/serializer/ArraySerializersTest.java b/java/fory-core/src/test/java/org/apache/fory/serializer/ArraySerializersTest.java index 891b5fe56a..2807acc31a 100644 --- a/java/fory-core/src/test/java/org/apache/fory/serializer/ArraySerializersTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/serializer/ArraySerializersTest.java @@ -35,7 +35,6 @@ import lombok.EqualsAndHashCode; import org.apache.fory.Fory; import org.apache.fory.ForyTestBase; -import org.apache.fory.annotation.ForyConstructor; import org.apache.fory.config.ForyBuilder; import org.apache.fory.config.Int64Encoding; import org.apache.fory.context.MetaReadContext; @@ -487,7 +486,6 @@ public void testArrayStructZeroCopy(Fory fory) { static class A { final int f1; - @ForyConstructor({"f1"}) A(int f1) { this.f1 = f1; } @@ -497,7 +495,6 @@ static class A { static class B extends A { final String f2; - @ForyConstructor({"f1", "f2"}) B(int f1, String f2) { super(f1); this.f2 = f2; @@ -521,7 +518,6 @@ public GenericArrayWrapper(Class clazz, int capacity) { this.array = (T[]) Array.newInstance(clazz, capacity); } - @ForyConstructor({"array"}) public GenericArrayWrapper(T[] array) { this.array = array; } diff --git a/java/fory-core/src/test/java/org/apache/fory/serializer/ObjectSerializerTest.java b/java/fory-core/src/test/java/org/apache/fory/serializer/ObjectSerializerTest.java index 15f0e2c490..265c246502 100644 --- a/java/fory-core/src/test/java/org/apache/fory/serializer/ObjectSerializerTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/serializer/ObjectSerializerTest.java @@ -26,29 +26,16 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; -import java.lang.reflect.Constructor; import java.nio.charset.StandardCharsets; -import java.util.AbstractList; -import java.util.ArrayList; -import java.util.List; import lombok.Data; import org.apache.fory.Fory; import org.apache.fory.ForyTestBase; import org.apache.fory.TestUtils; -import org.apache.fory.annotation.ForyConstructor; -import org.apache.fory.annotation.ForyField; import org.apache.fory.builder.CodecUtils; -import org.apache.fory.context.CopyContext; -import org.apache.fory.context.ReadContext; -import org.apache.fory.context.WriteContext; import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.memory.MemoryUtils; -import org.apache.fory.meta.TypeDef; import org.apache.fory.platform.AndroidSupport; -import org.apache.fory.platform.JdkVersion; -import org.apache.fory.resolver.RefMode; import org.apache.fory.test.bean.Cyclic; -import org.apache.fory.type.Types; import org.apache.fory.util.Preconditions; import org.testng.Assert; import org.testng.annotations.Test; @@ -186,155 +173,6 @@ public void testCopyCircularReference(Fory fory) { assertNotSame(cyclic1, cyclic); } - public static final class ConstructorCycle { - private final String name; - private ConstructorCycle next; - - @ForyConstructor("name") - public ConstructorCycle(String name) { - this.name = name; - } - } - - public static final class ConstructorCycleBeforeFinal { - @ForyField(id = 0) - private ConstructorCycleBeforeFinal next; - - @ForyField(id = 1) - private final String name; - - @ForyConstructor("name") - public ConstructorCycleBeforeFinal(String name) { - this.name = name; - } - } - - public static final class ConstructorOrder { - private int id; - private final String name; - - @ForyConstructor("name") - public ConstructorOrder(String name) { - this.name = name; - } - } - - public static final class ConstructorInterveningRef { - @ForyField(id = 0) - private Object first; - - @ForyField(id = 1) - private final String name; - - @ForyField(id = 2) - private Object second; - - @ForyConstructor("name") - public ConstructorInterveningRef(String name) { - this.name = name; - } - } - - public static final class ConstructorBackrefRoot { - private final ConstructorBackrefChild child; - - @ForyConstructor("child") - public ConstructorBackrefRoot(ConstructorBackrefChild child) { - this.child = child; - } - } - - public static final class ConstructorBackrefChild { - private ConstructorBackrefRoot root; - } - - public static final class ConstructorCustomRoot { - private final ConstructorCustomHolder holder; - - @ForyConstructor("holder") - public ConstructorCustomRoot(ConstructorCustomHolder holder) { - this.holder = holder; - } - } - - public static final class ConstructorCustomHolder { - private final String label; - private ConstructorCustomRoot root; - - public ConstructorCustomHolder(String label) { - this.label = label; - } - } - - public static final class ConstructorCustomHolderSerializer - extends Serializer { - public ConstructorCustomHolderSerializer(Fory fory) { - super(fory.getConfig(), ConstructorCustomHolder.class, true, false); - } - - @Override - public void write(WriteContext writeContext, ConstructorCustomHolder value) { - throw new UnsupportedOperationException(); - } - - @Override - public ConstructorCustomHolder read(ReadContext readContext) { - throw new UnsupportedOperationException(); - } - - @Override - public ConstructorCustomHolder copy(CopyContext copyContext, ConstructorCustomHolder value) { - return new ConstructorCustomHolder(value.label); - } - } - - public static final class ConstructorContainerBackrefRoot extends AbstractList { - private List self; - - @ForyConstructor("self") - public ConstructorContainerBackrefRoot(List self) { - this.self = self; - } - - @Override - public Object get(int index) { - return self.get(index); - } - - @Override - public int size() { - return self == null ? 0 : self.size(); - } - } - - public static final class RegisteredCtorBean { - @ForyField(id = 0) - private final String name; - - @ForyField(id = 1) - private final int age; - - private RegisteredCtorBean(int age, String name) { - this.name = name; - this.age = age; - } - } - - public static final class EmptyConstructorBinding { - private int id; - - @ForyConstructor({}) - public EmptyConstructorBinding() {} - - private EmptyConstructorBinding(int id) { - this.id = id; - } - } - - public static final class EmptyRegisteredCtorBean { - public EmptyRegisteredCtorBean() {} - } - public static final class FinalNoArgBean { private final int id; private final String name; @@ -356,7 +194,6 @@ public static final class FinalPostCtorBean { private final int id; private String label; - @ForyConstructor("label") public FinalPostCtorBean(String label) { id = -1; this.label = label; @@ -444,605 +281,6 @@ public void testFinalPostCtorCodegen() { assertEquals(newValue.label, value.label); } - @Test - public void testConstructorFieldProtocolOrder() { - ConstructorOrder value = new ConstructorOrder("root"); - value.id = 42; - Fory fory = - Fory.builder() - .withXlang(false) - .withRefTracking(true) - .withCodegen(false) - .withNumberCompressed(false) - .requireClassRegistration(false) - .build(); - ObjectSerializer serializer = - new ObjectSerializer<>(fory.getTypeResolver(), ConstructorOrder.class); - MemoryBuffer buffer = MemoryUtils.buffer(32); - withWriteContext(fory, buffer, context -> serializer.write(context, value)); - assertEquals(buffer.readInt32(), 42); - } - - @Test - public void testConstructorFieldProtocolOrderCodegen() { - ConstructorOrder value = new ConstructorOrder("root"); - value.id = 42; - Fory fory = - Fory.builder() - .withXlang(false) - .withRefTracking(true) - .withCodegen(true) - .withNumberCompressed(false) - .requireClassRegistration(false) - .build(); - Serializer serializer = - Serializers.newSerializer( - fory, - ConstructorOrder.class, - CodecUtils.loadOrGenObjectCodecClass(ConstructorOrder.class, fory)); - MemoryBuffer buffer = MemoryUtils.buffer(32); - withWriteContext(fory, buffer, context -> serializer.write(context, value)); - assertEquals(buffer.readInt32(), 42); - } - - @Test - public void testRegisterConstructor() throws Exception { - Constructor constructor = - RegisteredCtorBean.class.getDeclaredConstructor(int.class, String.class); - for (boolean codegen : new boolean[] {false, true}) { - Fory fory = - Fory.builder() - .withXlang(false) - .withRefTracking(true) - .withCodegen(codegen) - .requireClassRegistration(false) - .build(); - fory.registerConstructor(RegisteredCtorBean.class, constructor, "age", "name"); - assertEquals( - fory.getTypeResolver() - .getObjectCreator(RegisteredCtorBean.class) - .getConstructorFieldNames(), - new String[] {"age", "name"}); - RegisteredCtorBean value = new RegisteredCtorBean(42, "amy"); - RegisteredCtorBean newValue = (RegisteredCtorBean) fory.deserialize(fory.serialize(value)); - assertEquals(newValue.name, value.name); - assertEquals(newValue.age, value.age); - } - } - - @Test - public void testRegisterConstructorAfterSerializer() throws Exception { - Constructor constructor = - RegisteredCtorBean.class.getDeclaredConstructor(int.class, String.class); - Fory fory = - Fory.builder() - .withXlang(false) - .withRefTracking(true) - .withCodegen(false) - .requireClassRegistration(false) - .build(); - fory.getSerializer(RegisteredCtorBean.class); - Assert.assertThrows( - org.apache.fory.exception.ForyException.class, - () -> fory.registerConstructor(RegisteredCtorBean.class, constructor, "age", "name")); - } - - @Test - public void testEmptyRegisteredCtorRejected() throws Exception { - Constructor constructor = - EmptyRegisteredCtorBean.class.getDeclaredConstructor(); - Fory fory = - Fory.builder() - .withXlang(false) - .withRefTracking(true) - .withCodegen(false) - .requireClassRegistration(false) - .build(); - Assert.assertThrows( - org.apache.fory.exception.ForyException.class, - () -> fory.registerConstructor(EmptyRegisteredCtorBean.class, constructor)); - } - - @Test - public void testEmptyAnnotatedCtorRejected() { - Assert.assertThrows( - org.apache.fory.exception.ForyException.class, - () -> - new ObjectSerializer<>( - newConstructorBindingFory(false).getTypeResolver(), EmptyConstructorBinding.class)); - Assert.assertThrows( - org.apache.fory.exception.ForyException.class, - () -> { - Fory fory = newConstructorBindingFory(true); - TypeDef typeDef = fory.getTypeResolver().getTypeDef(EmptyConstructorBinding.class, true); - new CompatibleSerializer<>( - fory.getTypeResolver(), EmptyConstructorBinding.class, typeDef); - }); - Assert.assertThrows( - org.apache.fory.exception.ForyException.class, - () -> newConstructorBindingFory(false).copy(new EmptyConstructorBinding(1))); - } - - @Test - public void testCtorInterveningRef() { - Fory fory = - Fory.builder() - .withXlang(false) - .withRefTracking(true) - .withCodegen(false) - .requireClassRegistration(false) - .build(); - ConstructorInterveningRef newValue = - roundTripWithSerializer( - fory, - new ObjectSerializer<>(fory.getTypeResolver(), ConstructorInterveningRef.class), - newConstructorInterveningRef()); - assertInterveningRef(newValue); - } - - @Test - public void testCtorInterveningRefCodegen() { - Fory fory = - Fory.builder() - .withXlang(false) - .withRefTracking(true) - .withCodegen(true) - .requireClassRegistration(false) - .build(); - Serializer serializer = - Serializers.newSerializer( - fory, - ConstructorInterveningRef.class, - CodecUtils.loadOrGenObjectCodecClass(ConstructorInterveningRef.class, fory)); - ConstructorInterveningRef newValue = - roundTripWithSerializer(fory, serializer, newConstructorInterveningRef()); - assertInterveningRef(newValue); - } - - @Test - public void testCtorInterveningRefCompat() { - Fory fory = - Fory.builder() - .withXlang(false) - .withRefTracking(true) - .withCompatible(true) - .withCodegen(false) - .requireClassRegistration(false) - .build(); - TypeDef typeDef = fory.getTypeResolver().getTypeDef(ConstructorInterveningRef.class, true); - CompatibleSerializer serializer = - new CompatibleSerializer<>( - fory.getTypeResolver(), ConstructorInterveningRef.class, typeDef); - ConstructorInterveningRef newValue = - roundTripWithSerializer(fory, serializer, newConstructorInterveningRef()); - assertInterveningRef(newValue); - } - - @Test - public void testCtorInterveningRefCompatGen() { - Fory fory = - Fory.builder() - .withXlang(false) - .withRefTracking(true) - .withCompatible(true) - .withCodegen(true) - .requireClassRegistration(false) - .build(); - TypeDef typeDef = fory.getTypeResolver().getTypeDef(ConstructorInterveningRef.class, true); - Serializer serializer = - Serializers.newSerializer( - fory, - ConstructorInterveningRef.class, - CodecUtils.loadOrGenCompatibleCodecClass( - fory, ConstructorInterveningRef.class, typeDef)); - ConstructorInterveningRef newValue = - roundTripWithSerializer(fory, serializer, newConstructorInterveningRef()); - assertInterveningRef(newValue); - } - - @Test - public void testConstructorFieldBackrefRejected() { - if (JdkVersion.MAJOR_VERSION < 25) { - return; - } - ConstructorBackrefChild child = new ConstructorBackrefChild(); - ConstructorBackrefRoot value = new ConstructorBackrefRoot(child); - child.root = value; - Fory fory = - Fory.builder() - .withXlang(false) - .withRefTracking(true) - .withCodegen(false) - .requireClassRegistration(false) - .build(); - ObjectSerializer serializer = - new ObjectSerializer<>(fory.getTypeResolver(), ConstructorBackrefRoot.class); - MemoryBuffer buffer = MemoryUtils.buffer(32); - withWriteContext( - fory, - buffer, - context -> { - context.writeRefOrNull(value); - serializer.write(context, value); - }); - Assert.assertThrows( - org.apache.fory.exception.ForyException.class, - () -> - withReadContext( - fory, - buffer, - context -> { - byte tag = context.readRefOrNull(); - Preconditions.checkArgument(tag == Fory.REF_VALUE_FLAG); - context.preserveRefId(); - return serializer.read(context); - })); - } - - @Test - public void testContainerCtorBackrefRejected() { - if (JdkVersion.MAJOR_VERSION < 25) { - return; - } - ConstructorContainerBackrefRoot value = new ConstructorContainerBackrefRoot(null); - value.self = value; - Fory fory = - Fory.builder() - .withXlang(false) - .withRefTracking(true) - .withCodegen(false) - .requireClassRegistration(false) - .build(); - ObjectSerializer serializer = - new ObjectSerializer<>(fory.getTypeResolver(), ConstructorContainerBackrefRoot.class); - MemoryBuffer buffer = MemoryUtils.buffer(32); - withWriteContext( - fory, - buffer, - context -> { - context.writeRefOrNull(value); - serializer.write(context, value); - }); - Assert.assertThrows( - org.apache.fory.exception.ForyException.class, - () -> - withReadContext( - fory, - buffer, - context -> { - byte tag = context.readRefOrNull(); - Preconditions.checkArgument(tag == Fory.REF_VALUE_FLAG); - context.preserveRefId(); - return serializer.read(context); - })); - } - - @Test - public void testConstructorFieldCycle() { - ConstructorCycle value = new ConstructorCycle("root"); - value.next = value; - Fory fory = - Fory.builder() - .withXlang(false) - .withRefTracking(true) - .withCodegen(false) - .requireClassRegistration(false) - .build(); - ConstructorCycle newValue = - roundTripWithSerializer( - fory, new ObjectSerializer<>(fory.getTypeResolver(), ConstructorCycle.class), value); - assertEquals(newValue.name, value.name); - assertSame(newValue.next, newValue); - } - - @Test - public void testConstructorFieldCycleBeforeFinal() { - ConstructorCycleBeforeFinal value = new ConstructorCycleBeforeFinal("root"); - value.next = value; - Fory fory = - Fory.builder() - .withXlang(false) - .withRefTracking(true) - .withCodegen(false) - .requireClassRegistration(false) - .build(); - ConstructorCycleBeforeFinal newValue = - roundTripWithSerializer( - fory, - new ObjectSerializer<>(fory.getTypeResolver(), ConstructorCycleBeforeFinal.class), - value); - assertEquals(newValue.name, value.name); - assertSame(newValue.next, newValue); - } - - @Test - public void testConstructorFieldCycleCodegen() { - ConstructorCycle value = new ConstructorCycle("root"); - value.next = value; - Fory fory = - Fory.builder() - .withXlang(false) - .withRefTracking(true) - .withCodegen(true) - .requireClassRegistration(false) - .build(); - Serializer serializer = - Serializers.newSerializer( - fory, - ConstructorCycle.class, - CodecUtils.loadOrGenObjectCodecClass(ConstructorCycle.class, fory)); - ConstructorCycle newValue = roundTripWithSerializer(fory, serializer, value); - assertEquals(newValue.name, value.name); - assertSame(newValue.next, newValue); - } - - @Test - public void testCtorCycleBeforeFinalCodegen() { - ConstructorCycleBeforeFinal value = new ConstructorCycleBeforeFinal("root"); - value.next = value; - Fory fory = - Fory.builder() - .withXlang(false) - .withRefTracking(true) - .withCodegen(true) - .requireClassRegistration(false) - .build(); - Serializer serializer = - Serializers.newSerializer( - fory, - ConstructorCycleBeforeFinal.class, - CodecUtils.loadOrGenObjectCodecClass(ConstructorCycleBeforeFinal.class, fory)); - ConstructorCycleBeforeFinal newValue = roundTripWithSerializer(fory, serializer, value); - assertEquals(newValue.name, value.name); - assertSame(newValue.next, newValue); - } - - @Test - public void testConstructorFieldCycleCompatible() { - ConstructorCycle value = new ConstructorCycle("root"); - value.next = value; - Fory fory = - Fory.builder() - .withXlang(false) - .withRefTracking(true) - .withCompatible(true) - .withCodegen(true) - .requireClassRegistration(false) - .build(); - TypeDef typeDef = fory.getTypeResolver().getTypeDef(ConstructorCycle.class, true); - Serializer serializer = - Serializers.newSerializer( - fory, - ConstructorCycle.class, - CodecUtils.loadOrGenCompatibleCodecClass(fory, ConstructorCycle.class, typeDef)); - ConstructorCycle newValue = roundTripWithSerializer(fory, serializer, value); - assertEquals(newValue.name, value.name); - assertSame(newValue.next, newValue); - } - - @Test - public void testCtorCycleBeforeFinalCompat() { - ConstructorCycleBeforeFinal value = new ConstructorCycleBeforeFinal("root"); - value.next = value; - Fory fory = - Fory.builder() - .withXlang(false) - .withRefTracking(true) - .withCompatible(true) - .withCodegen(true) - .requireClassRegistration(false) - .build(); - TypeDef typeDef = fory.getTypeResolver().getTypeDef(ConstructorCycleBeforeFinal.class, true); - Serializer serializer = - Serializers.newSerializer( - fory, - ConstructorCycleBeforeFinal.class, - CodecUtils.loadOrGenCompatibleCodecClass( - fory, ConstructorCycleBeforeFinal.class, typeDef)); - ConstructorCycleBeforeFinal newValue = roundTripWithSerializer(fory, serializer, value); - assertEquals(newValue.name, value.name); - assertSame(newValue.next, newValue); - } - - @Test - public void testCtorCycleCompatNoCodegen() { - ConstructorCycleBeforeFinal value = new ConstructorCycleBeforeFinal("root"); - value.next = value; - Fory fory = - Fory.builder() - .withXlang(false) - .withRefTracking(true) - .withCompatible(true) - .withCodegen(false) - .requireClassRegistration(false) - .build(); - TypeDef typeDef = fory.getTypeResolver().getTypeDef(ConstructorCycleBeforeFinal.class, true); - CompatibleSerializer serializer = - new CompatibleSerializer<>( - fory.getTypeResolver(), ConstructorCycleBeforeFinal.class, typeDef); - ConstructorCycleBeforeFinal newValue = roundTripWithSerializer(fory, serializer, value); - assertEquals(newValue.name, value.name); - assertSame(newValue.next, newValue); - } - - @Test - public void testCtorBackrefCompatRejected() { - if (JdkVersion.MAJOR_VERSION < 25) { - return; - } - ConstructorBackrefChild child = new ConstructorBackrefChild(); - ConstructorBackrefRoot value = new ConstructorBackrefRoot(child); - child.root = value; - Fory fory = - Fory.builder() - .withXlang(false) - .withRefTracking(true) - .withCompatible(true) - .withCodegen(false) - .requireClassRegistration(false) - .build(); - TypeDef typeDef = fory.getTypeResolver().getTypeDef(ConstructorBackrefRoot.class, true); - CompatibleSerializer serializer = - new CompatibleSerializer<>(fory.getTypeResolver(), ConstructorBackrefRoot.class, typeDef); - MemoryBuffer buffer = MemoryUtils.buffer(32); - withWriteContext( - fory, - buffer, - context -> { - context.writeRefOrNull(value); - serializer.write(context, value); - }); - Assert.assertThrows( - org.apache.fory.exception.ForyException.class, - () -> - withReadContext( - fory, - buffer, - context -> { - byte tag = context.readRefOrNull(); - Preconditions.checkArgument(tag == Fory.REF_VALUE_FLAG); - context.preserveRefId(); - return serializer.read(context); - })); - } - - @Test - public void testContainerCtorBackrefCompatRejected() { - if (JdkVersion.MAJOR_VERSION < 25) { - return; - } - ConstructorContainerBackrefRoot value = new ConstructorContainerBackrefRoot(null); - value.self = value; - Fory fory = - Fory.builder() - .withXlang(false) - .withRefTracking(true) - .withCompatible(true) - .withCodegen(false) - .requireClassRegistration(false) - .build(); - TypeDef typeDef = - fory.getTypeResolver().getTypeDef(ConstructorContainerBackrefRoot.class, true); - CompatibleSerializer serializer = - new CompatibleSerializer<>( - fory.getTypeResolver(), ConstructorContainerBackrefRoot.class, typeDef); - MemoryBuffer buffer = MemoryUtils.buffer(32); - withWriteContext( - fory, - buffer, - context -> { - context.writeRefOrNull(value); - serializer.write(context, value); - }); - Assert.assertThrows( - org.apache.fory.exception.ForyException.class, - () -> - withReadContext( - fory, - buffer, - context -> { - byte tag = context.readRefOrNull(); - Preconditions.checkArgument(tag == Fory.REF_VALUE_FLAG); - context.preserveRefId(); - return serializer.read(context); - })); - } - - @Test - public void testCompatArrayCtorBackrefRejected() { - Object marker = new Object(); - Fory fory = - Fory.builder() - .withXlang(false) - .withRefTracking(true) - .withCodegen(false) - .requireClassRegistration(false) - .build(); - MemoryBuffer buffer = MemoryUtils.buffer(32); - withWriteContext( - fory, - buffer, - context -> { - context.writeRefOrNull(marker); - context.writeRefOrNull(marker); - }); - Assert.assertThrows( - org.apache.fory.exception.ForyException.class, - () -> - withReadContext( - fory, - buffer, - context -> { - byte tag = context.readRefOrNull(); - Preconditions.checkArgument(tag == Fory.REF_VALUE_FLAG); - context.preserveRefId(); - AbstractObjectSerializer.beginConstructorRef(context); - try { - Object value = - CompatibleCollectionArrayReader.read( - context, - RefMode.TRACKING, - CompatibleCollectionArrayReader.READ_ARRAY_TO_LIST, - Types.INT32_ARRAY, - Types.INT32, - List.class); - return AbstractObjectSerializer.ctorFieldValue( - context, value, ConstructorContainerBackrefRoot.class); - } finally { - AbstractObjectSerializer.endConstructorRef(context); - } - })); - } - - @Test(dataProvider = "foryCopyConfig") - public void testCtorBackrefCopyRejected(Fory fory) { - ConstructorBackrefChild child = new ConstructorBackrefChild(); - ConstructorBackrefRoot value = new ConstructorBackrefRoot(child); - child.root = value; - ObjectSerializer serializer = - new ObjectSerializer<>(fory.getTypeResolver(), ConstructorBackrefRoot.class); - Assert.assertThrows( - org.apache.fory.exception.ForyException.class, - () -> withCopyContext(fory, context -> serializer.copy(context, value))); - } - - @Test(dataProvider = "foryCopyConfig") - public void testContainerCtorBackrefCopyRejected(Fory fory) { - ConstructorContainerBackrefRoot value = new ConstructorContainerBackrefRoot(new ArrayList<>()); - value.self.add(value); - ObjectSerializer serializer = - new ObjectSerializer<>(fory.getTypeResolver(), ConstructorContainerBackrefRoot.class); - Assert.assertThrows( - org.apache.fory.exception.ForyException.class, - () -> withCopyContext(fory, context -> serializer.copy(context, value))); - } - - @Test(dataProvider = "foryCopyConfig") - public void testCtorCopyUsesSerializer(Fory fory) { - ConstructorCustomHolder holder = new ConstructorCustomHolder("custom"); - ConstructorCustomRoot value = new ConstructorCustomRoot(holder); - holder.root = value; - fory.registerSerializer( - ConstructorCustomHolder.class, new ConstructorCustomHolderSerializer(fory)); - ObjectSerializer serializer = - new ObjectSerializer<>(fory.getTypeResolver(), ConstructorCustomRoot.class); - ConstructorCustomRoot copy = withCopyContext(fory, context -> serializer.copy(context, value)); - assertNotSame(copy.holder, holder); - assertEquals(copy.holder.label, "custom"); - Assert.assertNull(copy.holder.root); - } - - @Test(dataProvider = "foryCopyConfig") - public void testConstructorFieldCycleCopy(Fory fory) { - ConstructorCycle value = new ConstructorCycle("root"); - value.next = value; - ObjectSerializer serializer = - new ObjectSerializer<>(fory.getTypeResolver(), ConstructorCycle.class); - ConstructorCycle newValue = withCopyContext(fory, context -> serializer.copy(context, value)); - assertEquals(newValue.name, value.name); - assertSame(newValue.next, newValue); - } - private static T roundTripWithSerializer(Fory fory, Serializer serializer, T value) { MemoryBuffer buffer = MemoryUtils.buffer(32); withWriteContext( @@ -1066,31 +304,6 @@ private static T roundTripWithSerializer(Fory fory, Serializer serializer return newValue; } - private static ConstructorInterveningRef newConstructorInterveningRef() { - ConstructorInterveningRef value = new ConstructorInterveningRef("root"); - Object shared = new String("shared"); - value.first = shared; - value.second = shared; - return value; - } - - private static void assertInterveningRef(ConstructorInterveningRef value) { - assertEquals(value.name, "root"); - assertEquals(value.first, "shared"); - assertSame(value.second, value.first); - Assert.assertNotSame(value.second, value); - } - - private static Fory newConstructorBindingFory(boolean compatible) { - return Fory.builder() - .withXlang(false) - .withRefTracking(true) - .withCompatible(compatible) - .withCodegen(false) - .requireClassRegistration(false) - .build(); - } - @Data public static class A { Integer f1; diff --git a/java/fory-core/src/test/java/org/apache/fory/serializer/ObjectStreamSerializerTest.java b/java/fory-core/src/test/java/org/apache/fory/serializer/ObjectStreamSerializerTest.java index eaccede8c1..fc0b7735e2 100644 --- a/java/fory-core/src/test/java/org/apache/fory/serializer/ObjectStreamSerializerTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/serializer/ObjectStreamSerializerTest.java @@ -29,7 +29,6 @@ import java.io.ObjectOutputStream; import java.io.ObjectStreamField; import java.io.Serializable; -import java.lang.reflect.Constructor; import java.lang.reflect.Method; import java.math.BigInteger; import java.net.Inet4Address; @@ -46,7 +45,6 @@ import lombok.EqualsAndHashCode; import org.apache.fory.Fory; import org.apache.fory.ForyTestBase; -import org.apache.fory.annotation.ForyConstructor; import org.apache.fory.config.ForyBuilder; import org.apache.fory.context.MetaReadContext; import org.apache.fory.context.MetaWriteContext; @@ -125,7 +123,6 @@ public static class AnnotatedObjectStreamType implements Serializable { int age; transient boolean readObjectCalled; - @ForyConstructor({"name", "age"}) public AnnotatedObjectStreamType(String name, int age) { this.name = name; this.age = age; @@ -260,11 +257,7 @@ private void readObject(ObjectInputStream s) throws IOException, ClassNotFoundEx } @Test(dataProvider = "javaFory") - public void testConstructorMappingRead(Fory fory) throws NoSuchMethodException { - Constructor constructor = - RegisteredObjectStreamType.class.getDeclaredConstructor(String.class, int.class); - fory.registerConstructor(RegisteredObjectStreamType.class, constructor, "name", "age"); - + public void testObjectStreamNoArgBypassRead(Fory fory) { AnnotatedObjectStreamType annotated = serDeCheckSerializer( fory, new AnnotatedObjectStreamType("annotated", 1), "ObjectStreamSerializer"); @@ -303,11 +296,7 @@ public void testInvalidParentCtor(Fory fory) { } @Test(dataProvider = "foryCopyConfig") - public void testConstructorMappingCopy(Fory fory) throws NoSuchMethodException { - Constructor constructor = - RegisteredObjectStreamType.class.getDeclaredConstructor(String.class, int.class); - fory.registerConstructor(RegisteredObjectStreamType.class, constructor, "name", "age"); - + public void testObjectStreamNoArgBypassCopy(Fory fory) { AnnotatedObjectStreamType copy = fory.copy(new AnnotatedObjectStreamType("copy", 3)); Assert.assertEquals(copy.name, "copy"); Assert.assertEquals(copy.age, 3); diff --git a/java/fory-core/src/test/java/org/apache/fory/serializer/collection/CollectionSerializersTest.java b/java/fory-core/src/test/java/org/apache/fory/serializer/collection/CollectionSerializersTest.java index ed762c7b3d..b2bf9029c7 100644 --- a/java/fory-core/src/test/java/org/apache/fory/serializer/collection/CollectionSerializersTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/serializer/collection/CollectionSerializersTest.java @@ -70,7 +70,6 @@ import lombok.EqualsAndHashCode; import org.apache.fory.Fory; import org.apache.fory.ForyTestBase; -import org.apache.fory.annotation.ForyConstructor; import org.apache.fory.context.ReadContext; import org.apache.fory.exception.DeserializationException; import org.apache.fory.exception.SerializationException; @@ -261,7 +260,6 @@ public void testSortedSet(boolean referenceTrackingConfig) { // Test serialize Comparator TreeSet set = new TreeSet<>( - (Comparator & Serializable) (s1, s2) -> { int delta = s1.length() - s2.length(); if (delta == 0) { @@ -963,7 +961,6 @@ public static class CollectionViewTestStruct { Collection collection; Set set; - @ForyConstructor({"collection", "set"}) public CollectionViewTestStruct(Collection collection, Set set) { this.collection = collection; this.set = set; @@ -1458,7 +1455,6 @@ public TestClassForDefaultCollectionSerializer() { this(new ArrayList<>()); } - @ForyConstructor({"data"}) public TestClassForDefaultCollectionSerializer(List data) { this.data = data; } diff --git a/java/fory-core/src/test/java/org/apache/fory/serializer/collection/MapSerializersTest.java b/java/fory-core/src/test/java/org/apache/fory/serializer/collection/MapSerializersTest.java index c4206e7397..34db014535 100644 --- a/java/fory-core/src/test/java/org/apache/fory/serializer/collection/MapSerializersTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/serializer/collection/MapSerializersTest.java @@ -54,7 +54,6 @@ import lombok.Data; import org.apache.fory.Fory; import org.apache.fory.ForyTestBase; -import org.apache.fory.annotation.ForyConstructor; import org.apache.fory.annotation.Ref; import org.apache.fory.collection.LazyMap; import org.apache.fory.collection.MapEntry; @@ -177,7 +176,6 @@ public void basicTestCaseWithMultiConfig( // testTreeMap TreeMap map = new TreeMap<>( - (Comparator & Serializable) (s1, s2) -> { int delta = s1.length() - s2.length(); if (delta == 0) { @@ -381,7 +379,6 @@ public void testTreeMap() { .build(); TreeMap map = new TreeMap<>( - (Comparator & Serializable) (s1, s2) -> { int delta = s1.length() - s2.length(); if (delta == 0) { @@ -401,7 +398,6 @@ public void testTreeMap() { public void testTreeMap(Fory fory) { TreeMap map = new TreeMap<>( - (Comparator & Serializable) (s1, s2) -> { int delta = s1.length() - s2.length(); if (delta == 0) { @@ -936,7 +932,6 @@ public TestClass1ForDefaultMap() { this(new HashSet<>()); } - @ForyConstructor({"data"}) public TestClass1ForDefaultMap(Set data) { this.data = data; } @@ -960,7 +955,6 @@ public TestClass2ForDefaultMap() { this(new HashSet<>()); } - @ForyConstructor({"data"}) public TestClass2ForDefaultMap(Set> data) { this.data = data; } @@ -1304,7 +1298,6 @@ public static class LazyMapCollectionFieldStruct { List> mapList; PrivateMap map; - @ForyConstructor({"mapList", "map"}) LazyMapCollectionFieldStruct( List> mapList, PrivateMap map) { this.mapList = mapList; @@ -1523,7 +1516,6 @@ public PrivateFinalMapFieldStruct() { this(new LinkedHashMap<>(), new LinkedHashMap<>()); } - @ForyConstructor({"valueMap", "keyMap"}) public PrivateFinalMapFieldStruct( Map valueMap, Map keyMap) { this.valueMap = valueMap; diff --git a/java/fory-format/pom.xml b/java/fory-format/pom.xml index 6cb04d742e..cc245465de 100644 --- a/java/fory-format/pom.xml +++ b/java/fory-format/pom.xml @@ -136,9 +136,24 @@ + + + + + + + - - + + + + + diff --git a/java/fory-latest-jdk-tests/src/test/java/org/apache/fory/integration_tests/ImmutableCollectionSerializersTest.java b/java/fory-latest-jdk-tests/src/test/java/org/apache/fory/integration_tests/ImmutableCollectionSerializersTest.java index 9d9878a445..c473dcaf7f 100644 --- a/java/fory-latest-jdk-tests/src/test/java/org/apache/fory/integration_tests/ImmutableCollectionSerializersTest.java +++ b/java/fory-latest-jdk-tests/src/test/java/org/apache/fory/integration_tests/ImmutableCollectionSerializersTest.java @@ -29,7 +29,6 @@ import lombok.Data; import org.apache.fory.Fory; import org.apache.fory.ThreadSafeFory; -import org.apache.fory.annotation.ForyConstructor; import org.apache.fory.exception.CopyException; import org.apache.fory.exception.SerializationException; import org.apache.fory.platform.JdkVersion; @@ -120,7 +119,6 @@ public void testSetFromMapIdentityJdk25() { public static class Pojo { List> data; - @ForyConstructor("data") public Pojo(List> data) { this.data = data; } diff --git a/java/fory-testsuite/src/test/java/org/apache/fory/serializer/collection/GuavaCollectionSerializersTest.java b/java/fory-testsuite/src/test/java/org/apache/fory/serializer/collection/GuavaCollectionSerializersTest.java index f961e2d2e0..b103a1467f 100644 --- a/java/fory-testsuite/src/test/java/org/apache/fory/serializer/collection/GuavaCollectionSerializersTest.java +++ b/java/fory-testsuite/src/test/java/org/apache/fory/serializer/collection/GuavaCollectionSerializersTest.java @@ -29,7 +29,6 @@ import java.util.Objects; import org.apache.fory.Fory; import org.apache.fory.TestBase; -import org.apache.fory.annotation.ForyConstructor; import org.testng.Assert; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; @@ -230,7 +229,6 @@ public void testNestedRefTrackingCopy(Fory fory) { public static final class Pojo { private final List> data; - @ForyConstructor("data") public Pojo(List> data) { this.data = data; } diff --git a/java/fory-testsuite/src/test/java/org/apache/fory/test/FastJsonTest.java b/java/fory-testsuite/src/test/java/org/apache/fory/test/FastJsonTest.java index 2c89a3bf3a..841d0b0028 100644 --- a/java/fory-testsuite/src/test/java/org/apache/fory/test/FastJsonTest.java +++ b/java/fory-testsuite/src/test/java/org/apache/fory/test/FastJsonTest.java @@ -24,7 +24,6 @@ import com.google.common.collect.Sets; import java.util.List; import org.apache.fory.Fory; -import org.apache.fory.annotation.ForyConstructor; import org.apache.fory.collection.Collections; import org.testng.Assert; import org.testng.annotations.DataProvider; @@ -35,7 +34,6 @@ public static class DemoResponse { private JSONObject json; private List objects; - @ForyConstructor("json") public DemoResponse(JSONObject json) { this.json = json; objects = Collections.ofArrayList(json); diff --git a/kotlin/fory-kotlin-ksp/src/main/kotlin/org/apache/fory/kotlin/ksp/ForyKotlinSymbolProcessor.kt b/kotlin/fory-kotlin-ksp/src/main/kotlin/org/apache/fory/kotlin/ksp/ForyKotlinSymbolProcessor.kt index 081b7941cb..827e4bec27 100644 --- a/kotlin/fory-kotlin-ksp/src/main/kotlin/org/apache/fory/kotlin/ksp/ForyKotlinSymbolProcessor.kt +++ b/kotlin/fory-kotlin-ksp/src/main/kotlin/org/apache/fory/kotlin/ksp/ForyKotlinSymbolProcessor.kt @@ -48,14 +48,6 @@ private data class ParsedStructFields( val fields: List, ) -private sealed class ConstructorFields { - object Absent : ConstructorFields() - - object Invalid : ConstructorFields() - - data class Present(val names: List) : ConstructorFields() -} - internal fun fieldLimitError(defaultFieldCount: Int): String? = if (defaultFieldCount > MAX_DEFAULT_FIELDS) { "Kotlin KSP xlang serializers currently support at most $MAX_DEFAULT_FIELDS defaulted constructor fields because Kotlin source generation must call constructors with omitted default arguments" @@ -108,27 +100,6 @@ internal fun ctorVisibilityError(modifiers: Set): String? = null } -internal fun constructorBindingError( - parameterNames: List, - fieldNames: List, - targetName: String, -): String? { - if (fieldNames.isEmpty()) { - return "@ForyConstructor on $targetName must declare at least one field name" - } - if (parameterNames.size != fieldNames.size) { - return "@ForyConstructor on $targetName must declare one field name for each primary constructor parameter" - } - val duplicates = fieldNames.groupingBy { it }.eachCount().filterValues { it > 1 }.keys - if (duplicates.isNotEmpty()) { - return "@ForyConstructor on $targetName declares duplicate field name ${duplicates.first()}" - } - if (fieldNames.any { it.isBlank() }) { - return "@ForyConstructor on $targetName must not declare blank field names" - } - return null -} - internal class ForyKotlinSymbolProcessor(private val environment: SymbolProcessorEnvironment) : SymbolProcessor { private val codeGenerator: CodeGenerator = environment.codeGenerator @@ -287,116 +258,39 @@ internal class ForyKotlinSymbolProcessor(private val environment: SymbolProcesso declaration: KSClassDeclaration, primaryConstructor: KSFunctionDeclaration, ): ParsedStructFields? { - val constructorFields = - when (val parsed = parseConstructorFields(declaration, primaryConstructor)) { - ConstructorFields.Absent -> null - ConstructorFields.Invalid -> return null - is ConstructorFields.Present -> parsed.names - } if (primaryConstructor.parameters.isEmpty()) { - if (constructorFields != null) { - logger.error( - "@ForyConstructor is only supported on Kotlin primary constructors that build schema fields", - primaryConstructor, - ) - return null - } return ParsedStructFields( KotlinStructConstruction.MUTABLE, parseVarFields(declaration) ?: return null, ) } - if (constructorFields == null) { - logger.error( - "Kotlin KSP constructor-backed @ForyStruct requires @ForyConstructor field mappings", - primaryConstructor, - ) - return null - } val propertiesByName = declaration.getAllProperties().associateBy { it.simpleName.asString() } - val fields = - parseCtorFields(declaration, primaryConstructor, propertiesByName, constructorFields) - ?: return null + val fields = parseCtorFields(declaration, primaryConstructor, propertiesByName) ?: return null return ParsedStructFields(KotlinStructConstruction.CONSTRUCTOR, fields) } - private fun parseConstructorFields( + private fun parseCtorFields( declaration: KSClassDeclaration, primaryConstructor: KSFunctionDeclaration, - ): ConstructorFields { - val annotation = - primaryConstructor.annotations.firstOrNull { isAnnotation(it, FORY_CONSTRUCTOR) } - ?: return ConstructorFields.Absent - val fieldNames = mutableListOf() - val argument = - annotation.arguments.firstOrNull { it.name?.asString() == "value" } - ?: annotation.arguments.singleOrNull() - when (val value = argument?.value) { - is String -> fieldNames.add(value) - is List<*> -> { - for (entry in value) { - if (entry !is String) { - logger.error("@ForyConstructor values must be field names", primaryConstructor) - return ConstructorFields.Invalid - } - fieldNames.add(entry) - } - } - is Array<*> -> { - for (entry in value) { - if (entry !is String) { - logger.error("@ForyConstructor values must be field names", primaryConstructor) - return ConstructorFields.Invalid - } - fieldNames.add(entry) - } - } - else -> { - logger.error("@ForyConstructor must declare field names", primaryConstructor) - return ConstructorFields.Invalid - } - } - val parameterNames = mutableListOf() + propertiesByName: Map, + ): List? { + val fields = mutableListOf() + val foryIds = hashSetOf() + var nextId = 0 for (parameter in primaryConstructor.parameters) { val parameterName = parameter.name?.asString() if (parameterName == null) { logger.error( - "@ForyConstructor requires named Kotlin primary constructor parameters", + "Kotlin KSP primary-constructor @ForyStruct requires named constructor parameters", parameter, ) - return ConstructorFields.Invalid + return null } - parameterNames.add(parameterName) - } - val bindingError = - constructorBindingError( - parameterNames, - fieldNames, - declaration.qualifiedName?.asString() ?: declaration.simpleName.asString(), - ) - if (bindingError != null) { - logger.error(bindingError, primaryConstructor) - return ConstructorFields.Invalid - } - return ConstructorFields.Present(fieldNames) - } - - private fun parseCtorFields( - declaration: KSClassDeclaration, - primaryConstructor: KSFunctionDeclaration, - propertiesByName: Map, - constructorFields: List, - ): List? { - val fields = mutableListOf() - val foryIds = hashSetOf() - var nextId = 0 - for ((index, parameter) in primaryConstructor.parameters.withIndex()) { - val parameterName = parameter.name?.asString() ?: continue - val fieldName = constructorFields[index] + val fieldName = parameterName val property = propertiesByName[fieldName] if (property == null) { logger.error( - "Constructor parameter $parameterName is not bound to an accessible schema property", + "Constructor parameter $parameterName is not declared as an accessible schema property", parameter ) return null @@ -407,7 +301,7 @@ internal class ForyKotlinSymbolProcessor(private val environment: SymbolProcesso val parameterTypeName = kotlinSourceTypeName(parameterType) if (fieldTypeName != parameterTypeName) { logger.error( - "@ForyConstructor field $fieldName type $fieldTypeName must match primary constructor parameter $parameterName type $parameterTypeName", + "Schema property $fieldName type $fieldTypeName must match primary constructor parameter $parameterName type $parameterTypeName", parameter, ) return null @@ -1836,7 +1730,6 @@ internal class ForyKotlinSymbolProcessor(private val environment: SymbolProcesso private companion object { const val FORY_STRUCT = "org.apache.fory.annotation.ForyStruct" - const val FORY_CONSTRUCTOR = "org.apache.fory.annotation.ForyConstructor" const val FORY_UNION = "org.apache.fory.annotation.ForyUnion" const val FORY_CASE = "org.apache.fory.annotation.ForyCase" const val FORY_UNKNOWN_CASE = "org.apache.fory.annotation.ForyUnknownCase" diff --git a/kotlin/fory-kotlin-ksp/src/main/kotlin/org/apache/fory/kotlin/ksp/KotlinSerializerSourceWriter.kt b/kotlin/fory-kotlin-ksp/src/main/kotlin/org/apache/fory/kotlin/ksp/KotlinSerializerSourceWriter.kt index dd33bb4b8d..e84d7b7b09 100644 --- a/kotlin/fory-kotlin-ksp/src/main/kotlin/org/apache/fory/kotlin/ksp/KotlinSerializerSourceWriter.kt +++ b/kotlin/fory-kotlin-ksp/src/main/kotlin/org/apache/fory/kotlin/ksp/KotlinSerializerSourceWriter.kt @@ -97,7 +97,6 @@ internal class KotlinSerializerSourceWriter(private val struct: KotlinSourceStru builder.append(" private val fieldsById: Array\n") builder.append(" private val constructorFieldIds: IntArray?\n") builder.append(" private val constructorFieldBits: LongArray?\n") - builder.append(" private val constructorFieldTypes: Array>?\n") builder.append(" private val classVersionHash: Int\n") builder.append(" private val sameSchemaCompatible: Boolean\n\n") } @@ -148,7 +147,6 @@ internal class KotlinSerializerSourceWriter(private val struct: KotlinSourceStru builder.append(" this.fieldsById = arrayOfNulls(0)\n") builder.append(" this.constructorFieldIds = null\n") builder.append(" this.constructorFieldBits = null\n") - builder.append(" this.constructorFieldTypes = null\n") builder.append(" this.classVersionHash = 0\n") builder.append(" this.sameSchemaCompatible = false\n") builder.append(" }\n\n") @@ -177,15 +175,21 @@ internal class KotlinSerializerSourceWriter(private val struct: KotlinSourceStru builder.append(" for (i in this.allFields.indices) {\n") builder.append(" this.fieldsById[this.allFieldIds[i]] = this.allFields[i]\n") builder.append(" }\n") - builder.append( - " this.constructorFieldIds = if (objectCreator.hasConstructorFields()) buildConstructorFieldIds(DESCRIPTORS) else null\n" - ) + if (struct.construction == KotlinStructConstruction.CONSTRUCTOR) { + builder.append(" this.constructorFieldIds = intArrayOf(") + for (i in struct.fields.indices) { + if (i > 0) { + builder.append(", ") + } + builder.append(struct.fields[i].id) + } + builder.append(")\n") + } else { + builder.append(" this.constructorFieldIds = null\n") + } builder.append( " this.constructorFieldBits = buildConstructorFieldBits(DESCRIPTORS.size, constructorFieldIds)\n" ) - builder.append( - " this.constructorFieldTypes = if (constructorFieldIds != null) constructorFieldTypes() else null\n" - ) writeScalarBindings() builder.append( " this.classVersionHash = if (typeResolver.checkClassVersion()) computeClassVersionHash(DESCRIPTORS) else 0\n" @@ -376,10 +380,9 @@ internal class KotlinSerializerSourceWriter(private val struct: KotlinSourceStru .append(" private fun newConstructorObject(fieldValues: Array): ") .append(struct.typeName) .append(" {\n") - builder.append( - " return objectCreator.newInstanceWithArguments(*constructorArgs(fieldValues, constructorFieldIds!!, constructorFieldTypes!!)) as " - ) - builder.append(struct.typeName).append("\n") + builder.append(" return ") + appendFieldValuesConstructorCall() + builder.append("\n") builder.append(" }\n\n") builder.append( @@ -468,40 +471,12 @@ internal class KotlinSerializerSourceWriter(private val struct: KotlinSourceStru builder.append(" if (typeResolver.checkClassVersion()) {\n") builder.append(" checkClassVersion(buffer.readInt32(), classVersionHash)\n") builder.append(" }\n") - builder.append(" if (constructorFieldIds != null) {\n") - builder.append(" return readSchemaConstructor(readContext)\n") - builder.append(" }\n") if (struct.construction == KotlinStructConstruction.MUTABLE) { writeMutableReadBody() builder.append(" }\n\n") - writeConstructorRead() return } - writeLocalDeclarations() - builder.append(" for (i in allFields.indices) {\n") - builder.append(" val fieldInfo = allFields[i]\n") - builder.append(" when (allFieldIds[i]) {\n") - for (field in struct.fields) { - builder.append(" ").append(field.id).append(" -> ") - val direct = directReadExpression(field) - if (direct == null) { - builder - .append(field.localName) - .append(" = ") - .append(castReadExpression(field, "readFieldValue(readContext, fieldInfo)")) - .append("\n") - } else { - builder.append(field.localName).append(" = ").append(direct).append("\n") - } - } - builder.append( - " else -> throw IllegalStateException(\"Unknown generated field id \${allFieldIds[i]}\")\n" - ) - builder.append(" }\n") - builder.append(" }\n") - builder.append(" return ") - appendConstructorCall(defaultMask = 0L) - builder.append("\n") + builder.append(" return readSchemaConstructor(readContext)\n") builder.append(" }\n\n") writeConstructorRead() } @@ -812,9 +787,9 @@ internal class KotlinSerializerSourceWriter(private val struct: KotlinSourceStru return } if (struct.construction == KotlinStructConstruction.CONSTRUCTOR) { - builder.append(" if (constructorFieldIds != null) {\n") - builder.append(" return readCompatibleConstructor(readContext)\n") - builder.append(" }\n") + builder.append(" return readCompatibleConstructor(readContext)\n") + builder.append(" }\n\n") + return } if (struct.construction == KotlinStructConstruction.MUTABLE) { writeMutableCompatibleReadBody() @@ -1010,57 +985,12 @@ internal class KotlinSerializerSourceWriter(private val struct: KotlinSourceStru builder.append(" if (immutable) {\n") builder.append(" return value\n") builder.append(" }\n") - builder.append(" if (constructorFieldIds != null) {\n") - builder.append(" return copyConstructorObject(copyContext, value)\n") - builder.append(" }\n") if (struct.construction == KotlinStructConstruction.MUTABLE) { writeMutableCopyBody() builder.append(" }\n") return } - for (field in struct.fields) { - if (isDirectCopyValue(field.type)) { - builder - .append(" val ") - .append(field.localName) - .append(" = value.") - .append(field.name) - .append("\n") - } else if (isDenseUnsignedArray(field)) { - builder.append(" val ").append(field.localName).append(" = value.").append(field.name) - if (field.nullable) { - builder.append("?.copyOf()\n") - } else { - builder.append(".copyOf()\n") - } - } else if (field.type.isCollectionOrMap()) { - builder - .append(" val ") - .append(field.localName) - .append(": ") - .append(localVariableType(field)) - .append(" = ") - .append(copyContainerExpression(field.type, "value.${field.name}", 0)) - .append("\n") - } else { - builder - .append(" val ") - .append(field.localName) - .append(": ") - .append(localVariableType(field)) - .append(" = ") - .append( - castReadExpression( - field, - "copyFieldValue(copyContext, value.${field.name}, fieldsById[${field.id}]!!)" - ) - ) - .append("\n") - } - } - builder.append(" return ") - appendConstructorCall(defaultMask = 0L) - builder.append("\n") + builder.append(" return copyConstructorObject(copyContext, value)\n") builder.append(" }\n") } @@ -1204,6 +1134,30 @@ internal class KotlinSerializerSourceWriter(private val struct: KotlinSourceStru builder.append(")") } + private fun appendFieldValuesConstructorCall() { + builder.append(struct.typeName).append("(") + var first = true + for (field in struct.fields) { + if (!first) { + builder.append(", ") + } + builder + .append(field.constructorParameterName) + .append(" = ") + .append(constructorFieldValueExpression(field)) + first = false + } + builder.append(")") + } + + private fun constructorFieldValueExpression(field: KotlinSourceField): String { + val source = "fieldValues[${field.id}]" + if (field.type.valueTypeName == "Any?") { + return source + } + return "($source as ${field.type.valueTypeName})" + } + private fun constructorValueExpression(field: KotlinSourceField): String { val localValue = if (field.nullable || field.type.primitive || isScalarUnsigned(field)) { diff --git a/kotlin/fory-kotlin-ksp/src/test/kotlin/org/apache/fory/kotlin/ksp/ProcessorValidationTest.kt b/kotlin/fory-kotlin-ksp/src/test/kotlin/org/apache/fory/kotlin/ksp/ProcessorValidationTest.kt index a0253036b7..3498fdf932 100644 --- a/kotlin/fory-kotlin-ksp/src/test/kotlin/org/apache/fory/kotlin/ksp/ProcessorValidationTest.kt +++ b/kotlin/fory-kotlin-ksp/src/test/kotlin/org/apache/fory/kotlin/ksp/ProcessorValidationTest.kt @@ -77,28 +77,7 @@ class ProcessorValidationTest { } @Test - fun validatesConstructorBinding() { - assertNull(constructorBindingError(listOf("userName"), listOf("name"), "example.User")) - assertEquals( - constructorBindingError(listOf("name"), listOf("name", "age"), "example.User"), - "@ForyConstructor on example.User must declare one field name for each primary constructor parameter", - ) - assertEquals( - constructorBindingError(emptyList(), emptyList(), "example.User"), - "@ForyConstructor on example.User must declare at least one field name", - ) - assertEquals( - constructorBindingError(listOf("name", "age"), listOf("name", "name"), "example.User"), - "@ForyConstructor on example.User declares duplicate field name name", - ) - assertEquals( - constructorBindingError(listOf("name"), listOf(""), "example.User"), - "@ForyConstructor on example.User must not declare blank field names", - ) - } - - @Test - fun constructorBindingNamesArguments() { + fun constructorNamesArguments() { val stringType = KotlinSourceTypeNode( rawClassExpression = "String::class.java", @@ -142,10 +121,9 @@ class ProcessorValidationTest { .write() assertTrue(source.contains("writeContext.writeString(value.name)")) - assertTrue(source.contains("return User(userName = field0!!)")) + assertTrue(source.contains("this.constructorFieldIds = intArrayOf(0)")) + assertTrue(source.contains("return User(userName = (fieldValues[0] as String))")) assertTrue(!source.contains("return User(name = field0!!)")) - assertTrue(source.contains("constructorFieldIds = if (objectCreator.hasConstructorFields())")) - assertTrue(source.contains("objectCreator.newInstanceWithArguments(*constructorArgs")) assertTrue(source.contains("fieldValues[0] = value.name")) assertFalse(source.contains("copyConstructorFieldValue(copyContext, value, value.name")) assertFalse(source.contains("NATURAL_ORDER_COMPARATOR")) @@ -484,7 +462,7 @@ class ProcessorValidationTest { assertTrue( source.contains( - "return User(counts = KotlinCollectionAdapters.toTreeMap(field0!!), names = run { val readSource0 = (field1!! as Collection<*>);" + "return User(counts = (fieldValues[0] as kotlin.collections.Map), names = (fieldValues[1] as kotlin.collections.List>), arrays = (fieldValues[2] as kotlin.collections.List), nestedCounts = (fieldValues[3] as java.util.TreeMap>))" ) ) assertTrue( @@ -512,7 +490,7 @@ class ProcessorValidationTest { ) assertTrue( source.contains( - "nestedCounts = run { val readSource0 = (field3!! as Map<*, *>); val readTarget0 = java.util.TreeMap();" + "3 -> run { val readSource0 = ((readCompatibleFieldValue(readContext, remoteField, localField) as java.util.TreeMap>) as Map<*, *>); val readTarget0 = java.util.TreeMap();" ) ) assertTrue( @@ -627,6 +605,7 @@ class ProcessorValidationTest { qualifiedTypeName = "example.WideStruct", serializerName = "WideStruct_ForySerializer", serializerVisibility = KotlinSerializerVisibility.PUBLIC, + construction = KotlinStructConstruction.MUTABLE, fields = (0 until 70).map { id -> KotlinSourceField( diff --git a/kotlin/fory-kotlin-tests/src/main/kotlin/org/apache/fory/kotlin/xlang/KotlinXlangPeer.kt b/kotlin/fory-kotlin-tests/src/main/kotlin/org/apache/fory/kotlin/xlang/KotlinXlangPeer.kt index 470a159f73..3f3504af12 100644 --- a/kotlin/fory-kotlin-tests/src/main/kotlin/org/apache/fory/kotlin/xlang/KotlinXlangPeer.kt +++ b/kotlin/fory-kotlin-tests/src/main/kotlin/org/apache/fory/kotlin/xlang/KotlinXlangPeer.kt @@ -34,7 +34,6 @@ import org.apache.fory.BaseFory import org.apache.fory.Fory import org.apache.fory.annotation.ArrayType import org.apache.fory.annotation.ForyCase -import org.apache.fory.annotation.ForyConstructor import org.apache.fory.annotation.ForyField import org.apache.fory.annotation.ForyStruct import org.apache.fory.annotation.ForyUnion @@ -58,24 +57,14 @@ import org.apache.fory.type.union.UnknownCase @ForyStruct public data class KotlinUser -@ForyConstructor("id", "name", "score") constructor( @ForyField(id = 1) val id: @Fixed UInt, @ForyField(id = 2) val name: String = "anonymous", @ForyField(id = 3) val score: @VarInt Long, ) -@ForyStruct -public data class KotlinRegisteredSwap -@ForyConstructor("left", "right") -constructor( - @ForyField(id = 1) val left: String, - @ForyField(id = 2) val right: String, -) - @ForyStruct internal data class KotlinInternalUser -@ForyConstructor("id", "name") constructor( @ForyField(id = 1) val id: UInt, @ForyField(id = 2) val name: String = "internal", @@ -83,17 +72,6 @@ constructor( @ForyStruct public data class KotlinConcreteCollections -@ForyConstructor( - "names", - "values", - "tags", - "counts", - "mutableNames", - "mutableTags", - "mutableCounts", - "sortedNames", - "concurrentCounts", -) constructor( @ForyField(id = 1) val names: ArrayList, @ForyField(id = 2) val values: java.util.LinkedList, @@ -108,7 +86,6 @@ constructor( @ForyStruct public data class KotlinUnsignedCollections -@ForyConstructor("ids", "optionalIds", "totals", "byName", "namesById") constructor( @ForyField(id = 1) val ids: List, @ForyField(id = 2) val optionalIds: List, @@ -119,21 +96,6 @@ constructor( @ForyStruct public data class KotlinSchemaSurface -@ForyConstructor( - "nullableNames", - "dynamicList", - "dynamicValues", - "bytesAsArray", - "bits", - "unsignedLongs", - "fieldSiteId", - "denseIds", - "noRefUser", - "noRefUsers", - "chunks", - "chunksByName", - "nestedSortedNames", -) constructor( @ForyField(id = 1) val nullableNames: List?, @ForyField(id = 2) val dynamicList: List<*>, @@ -152,20 +114,6 @@ constructor( @ForyStruct public data class KotlinDenseArrays -@ForyConstructor( - "ubytes", - "ushorts", - "uints", - "ulongs", - "ints", - "longs", - "bytes", - "shorts", - "floats", - "doubles", - "booleans", - "nullableUInts", -) constructor( @ForyField(id = 1) val ubytes: UByteArray, @ForyField(id = 2) val ushorts: UShortArray, @@ -183,12 +131,10 @@ constructor( @ForyStruct public data class KotlinNullableCompatibleWriter -@ForyConstructor("anchor") constructor(@ForyField(id = 1) val anchor: String) @ForyStruct public data class KotlinNullableCompatibleReader -@ForyConstructor("anchor", "maybeBoolean", "maybeInt", "maybeLong", "maybeUInt", "maybeULong") constructor( @ForyField(id = 1) val anchor: String, @ForyField(id = 2) val maybeBoolean: Boolean?, @@ -200,12 +146,10 @@ constructor( @ForyStruct public data class KotlinDefaultCompatibleWriter -@ForyConstructor("id") constructor(@ForyField(id = 1) val id: Int) @ForyStruct public data class KotlinDefaultCompatibleReader -@ForyConstructor("id", "name") constructor( @ForyField(id = 1) val id: Int, @ForyField(id = 2) val name: String = "generated-default", @@ -213,7 +157,6 @@ constructor( @ForyStruct public data class KotlinDefaultRefWriter -@ForyConstructor("id", "next") constructor( @ForyField(id = 1) val id: Int, @Ref @ForyField(id = 2) var next: KotlinDefaultRefWriter?, @@ -221,7 +164,6 @@ constructor( @ForyStruct public data class KotlinDefaultRefReader -@ForyConstructor("id", "name", "next") constructor( @ForyField(id = 1) val id: Int, @ForyField(id = 3) val name: String = "generated-default", @@ -230,16 +172,6 @@ constructor( @ForyStruct public data class KotlinDurationAndHalfArrays -@ForyConstructor( - "duration", - "date", - "instant", - "decimal", - "float16", - "bfloat16", - "float16s", - "bfloat16s", -) constructor( @ForyField(id = 1) val duration: kotlin.time.Duration, @ForyField(id = 2) val date: LocalDate, @@ -260,7 +192,6 @@ public class KotlinMutableNode() { @ForyStruct public class KotlinCtorBackrefRoot -@ForyConstructor("child") constructor(@ForyField(id = 1) val child: KotlinCtorBackrefChild) @ForyStruct @@ -295,14 +226,7 @@ private fun staticSerializerRoundTrip(dataFile: String) { checkNoArgRegisterReceivers() val fory = newFory() - fory.registerConstructor( - KotlinRegisteredSwap::class.java, - KotlinRegisteredSwap::class.java.getDeclaredConstructor(String::class.java, String::class.java), - "right", - "left", - ) fory.register("kotlin", "KotlinUser") - fory.register("kotlin", "KotlinRegisteredSwap") fory.register("kotlin", "KotlinInternalUser") fory.register("kotlin", "KotlinConcreteCollections") fory.register("kotlin", "KotlinUnsignedCollections") @@ -331,16 +255,6 @@ private fun staticSerializerRoundTrip(dataFile: String) { check(descriptors[0].typeRef.typeExtMeta.typeId() == Types.UINT32) check(descriptors[2].typeRef.typeExtMeta.typeId() == Types.VARINT64) - val swap = KotlinRegisteredSwap(left = "left", right = "right") - val swapped = KotlinRegisteredSwap(left = "right", right = "left") - check(fory.deserialize(fory.serialize(swap), KotlinRegisteredSwap::class.java) == swapped) - check(fory.copy(swap) == swapped) - check( - fory.getSerializer(KotlinRegisteredSwap::class.java) is StaticGeneratedStructSerializer<*> - ) { - "KotlinRegisteredSwap did not load a static generated serializer" - } - val internalUser = KotlinInternalUser(id = UInt.MAX_VALUE, name = "internal-static") check( fory.deserialize(fory.serialize(internalUser), KotlinInternalUser::class.java) == internalUser diff --git a/kotlin/fory-kotlin/src/main/kotlin/org/apache/fory/serializer/kotlin/KotlinDefaultValueSupport.kt b/kotlin/fory-kotlin/src/main/kotlin/org/apache/fory/serializer/kotlin/KotlinDefaultValueSupport.kt index 26850ff5d6..3cd17bdff7 100644 --- a/kotlin/fory-kotlin/src/main/kotlin/org/apache/fory/serializer/kotlin/KotlinDefaultValueSupport.kt +++ b/kotlin/fory-kotlin/src/main/kotlin/org/apache/fory/serializer/kotlin/KotlinDefaultValueSupport.kt @@ -172,7 +172,7 @@ internal class KotlinDefaultValueSupport : DefaultValueUtils.DefaultValueSupport seen[clazz] = true try { if (!AndroidSupport.IS_ANDROID) { - return newDefaultInstance(clazz, seen) + return newDefaultInstance(clazz) } return newPublicDefaultInstance(clazz, seen) } finally { @@ -180,17 +180,8 @@ internal class KotlinDefaultValueSupport : DefaultValueUtils.DefaultValueSupport } } - private fun newDefaultInstance(clazz: Class<*>, seen: IdentityHashMap, Boolean>): Any? { - val creator = ObjectCreators.getObjectCreator(clazz) - if (!creator.hasConstructorFields()) { - return creator.newInstance() - } - val parameterTypes = creator.getConstructorFieldTypes() - val args = arrayOfNulls(parameterTypes.size) - for (i in parameterTypes.indices) { - args[i] = getDefaultValueForType(parameterTypes[i], seen) ?: return null - } - return creator.newInstanceWithArguments(*args) + private fun newDefaultInstance(clazz: Class<*>): Any? { + return ObjectCreators.getObjectCreator(clazz).newInstance() } private fun newPublicDefaultInstance( From 20ed6c85e7fece254c5be89d1672d4705a006862 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Tue, 2 Jun 2026 22:31:23 +0800 Subject: [PATCH 60/69] refactor(java): clean object instantiator ownership --- .agents/languages/java.md | 37 ++-- .../jpms_tests/src/main/java/module-info.java | 1 - .../constructor/PrivateConstructorBean.java | 42 ---- .../JpmsFieldAccessorTest.java | 12 -- .../processing/ForyStructProcessorTest.java | 3 +- java/fory-core/pom.xml | 10 +- .../fory/builder/BaseObjectCodecBuilder.java | 18 +- .../org/apache/fory/builder/CodecBuilder.java | 27 +-- .../fory/builder/ObjectCodecBuilder.java | 5 +- .../builder/StaticCompatibleCodecBuilder.java | 3 +- .../org/apache/fory/codegen/Expression.java | 16 +- ...ctCreator.java => ObjectInstantiator.java} | 18 +- ...Creators.java => ObjectInstantiators.java} | 184 +++++++++--------- ...tor.java => UnsafeObjectInstantiator.java} | 19 +- .../apache/fory/resolver/ClassResolver.java | 6 +- .../apache/fory/resolver/SharedRegistry.java | 30 ++- .../apache/fory/resolver/TypeResolver.java | 13 +- .../serializer/AbstractObjectSerializer.java | 19 +- .../CompatibleLayerSerializerBase.java | 17 +- .../fory/serializer/CompatibleSerializer.java | 2 +- .../fory/serializer/ExceptionSerializers.java | 36 ++-- .../serializer/ExternalizableSerializer.java | 2 +- .../fory/serializer/ObjectSerializer.java | 10 +- .../serializer/ObjectStreamSerializer.java | 10 +- .../serializer/SerializationHookLookup.java | 52 ++--- .../collection/ChildContainerSerializers.java | 17 +- .../reflect/ConstructorBypassAllocator.java | 99 ---------- .../reflect/UnsafeObjectInstantiator.java | 49 +++++ .../fory-core/native-image.properties | 18 +- .../test/java/org/apache/fory/TestUtils.java | 4 +- .../builder/Jdk25MultiReleaseJarVerifier.java | 6 +- ...Test.java => ObjectInstantiatorsTest.java} | 32 +-- .../fory/reflect/ReflectionUtilsTest.java | 2 +- .../fory/serializer/ObjectSerializerTest.java | 1 - .../collection/CollectionSerializersTest.java | 16 +- .../collection/MapSerializersTest.java | 48 ++--- .../ksp/KotlinSerializerSourceWriter.kt | 7 +- .../kotlin/ksp/ProcessorValidationTest.kt | 4 +- .../kotlin/KotlinDefaultValueSupport.kt | 4 +- 39 files changed, 416 insertions(+), 483 deletions(-) delete mode 100644 integration_tests/jpms_tests/src/main/java/org/apache/fory/integration_tests/constructor/PrivateConstructorBean.java rename java/fory-core/src/main/java/org/apache/fory/reflect/{ObjectCreator.java => ObjectInstantiator.java} (78%) rename java/fory-core/src/main/java/org/apache/fory/reflect/{ObjectCreators.java => ObjectInstantiators.java} (64%) rename java/fory-core/src/main/java/org/apache/fory/reflect/{ConstructorBypassAllocator.java => UnsafeObjectInstantiator.java} (82%) delete mode 100644 java/fory-core/src/main/java25/org/apache/fory/reflect/ConstructorBypassAllocator.java create mode 100644 java/fory-core/src/main/java25/org/apache/fory/reflect/UnsafeObjectInstantiator.java rename java/fory-core/src/test/java/org/apache/fory/reflect/{ObjectCreatorsTest.java => ObjectInstantiatorsTest.java} (74%) diff --git a/.agents/languages/java.md b/.agents/languages/java.md index 50f236ff16..27a08f8985 100644 --- a/.agents/languages/java.md +++ b/.agents/languages/java.md @@ -63,26 +63,37 @@ Load this file when changing anything under `java/` or when Java drives a cross- - Do not probe JDK25+ trusted-lookup availability and turn `_JDKAccess` field-access booleans false when the required `java.base/java.lang.invoke` open is missing. Keep those access flags true on JDK25+ and let the owning trusted-lookup path raise the configuration error. - Keep JDK25+ unsafe-removal implementation invariants in agent/design docs and tests, not user guides. User guides should document user actions such as `--sun-misc-unsafe-memory-access=deny`, `java.base/java.lang.invoke` opens, and package opens for user named modules; do not expose internal serializer names, owner-model rationale, or avoided fallback strategies there. - JDK25+ zero-Unsafe final-field writes must use a true target-class trusted lookup from the original `IMPL_LOOKUP`, not `IMPL_LOOKUP.in(type)`. JDK26+ normal Fory final-field restoration must pass with `--illegal-final-field-mutation=deny` and must not require `--enable-final-field-mutation`. -- For JDK25+ object creation, do not use `sun.reflect.ReflectionFactory` or `jdk.internal.reflect.ReflectionFactory`. The shared `ObjectCreators` facade should route ObjectStream-compatible construction through `ParentNoArgCtrObjectCreator`; the replaceable constructor-bypass allocator owns the JDK8-24 Unsafe allocation path and the JDK25+ `ObjectStreamClass.newInstance` path. Classes without a no-arg constructor may use that trusted-lookup owner; unsupported classes require a record canonical constructor path or a custom serializer. -- In JDK25+ constructor-bypass allocation, the allocator is per type: pass the target class to the allocator constructor, cache `ObjectStreamClass.lookupAny(...)` in that instance, and expose an instance `allocate()` method. Let `ObjectStreamClass.newInstance` validate unsupported classes. Do not add redundant `Serializable` prechecks or per-call descriptor lookups, and keep exception/message construction in cold helper methods. +- For JDK25+ object creation, do not use `sun.reflect.ReflectionFactory`, + `jdk.unsupported`, `ObjectStreamClass.newInstance`, or an Unsafe-backed object instantiator. + The shared `ObjectInstantiators` facade should route supported no-constructor construction + through `ParentNoArgCtrInstantiator`, whose JDK25+ path owns trusted-lookup access to + `jdk.internal.reflect.ReflectionFactory` in `java.base`. This must not require + `--add-opens=java.base/jdk.internal.reflect=...`; the only JDK25+ platform open remains + `java.base/java.lang.invoke=org.apache.fory.core`. Classes unsupported by the serialization + constructor model require an accessible no-arg constructor, a record canonical constructor path, + or a custom serializer. +- `UnsafeObjectInstantiator` is the JDK8-24 Unsafe owner only. It must be a top-level instantiator + with a Java25 multi-release stub that contains no Unsafe, ObjectStream, ReflectionFactory, or + constructor-bypass implementation. - Keep the Java25 `_Lookup` overlay unless a future refactor can merge it without exposing Unsafe to the JDK25 class graph. Root `_Lookup` uses Unsafe for the JDK8-24 trusted-lookup fast path, while Java25 `_Lookup` uses the required `java.lang.invoke` open. `DefineClass` is root-owned; when Java25+ generated serializers need hidden nestmate class definition, it must use cached method handles and reflective `Lookup.ClassOption.NESTMATE` loading so Java 8 through Java 14 can still load the root class safely. - Treat `ByteArrayOutputStream` and `ByteArrayInputStream` as ordinary streams on every JDK. Do not restore private-buffer wrapping for JDK8-24 performance, because that reintroduces `java.base/java.io` private-field ownership and module-open requirements. -- `ObjectStreamSerializer` must use its stream-specific object creator path and must not use - `TypeResolver.getObjectCreator`. ObjectStream reconstruction creates the object before stream +- `ObjectStreamSerializer` must use its stream-specific object-instantiator path and must not use + `TypeResolver.getObjectInstantiator`. ObjectStream reconstruction creates the object before stream fields are read. This path must not invoke Serializable class constructors, including no-arg constructors; the JDK8-24 ReflectionFactory template constructor must be the first non-Serializable superclass no-arg constructor, and private no-arg constructors must be rejected instead of made accessible. Package-private no-arg constructors are valid only for same-package Serializable subclasses. -- Runtime object creator caches are `SharedRegistry` state backed by `ConcurrentHashMap`. - Keep ObjectStream-specific creators separate from normal object creators. -- Generated Fory object serializers must initialize object-creator fields through - `TypeResolver.getObjectCreator(Class)`, so generated code respects runtime-scoped object creators. +- Runtime object-instantiator caches are `SharedRegistry` state backed by `ConcurrentHashMap`. + Keep ObjectStream-specific instantiators separate from normal object instantiators. +- Generated Fory object serializers must initialize object-instantiator fields through + `TypeResolver.getObjectInstantiator(Class)`, so generated code respects runtime-scoped object + instantiators. Do not emit generated calls to - `ObjectCreators.getObjectCreator(TypeResolver, Class)` or bypass the runtime-scoped owner; format - builders without a Fory runtime context may use the base `ObjectCreators.getObjectCreator(Class)` + `ObjectInstantiators.getObjectInstantiator(TypeResolver, Class)` or bypass the runtime-scoped owner; format + builders without a Fory runtime context may use the base `ObjectInstantiators.getObjectInstantiator(Class)` construction default. - Root codegen and builder classes that still need Unsafe on JDK8-24 must route symbolic Unsafe access through a helper with a Java 25 replacement. Do not leave `_JDKAccess.unsafe()` or @@ -106,7 +117,7 @@ Load this file when changing anything under `java/` or when Java drives a cross- primitives. - JDK25+ serialization hook access must use the required trusted lookup from `java.base/java.lang.invoke=org.apache.fory.core`. Keep `sun.reflect.ReflectionFactory` as a - JDK8-24 optimization only, and do not add per-type reflective escapes for hook invocation. + JDK8-24 hook optimization only, and do not add per-type reflective escapes for hook invocation. - JDK25+ `PlatformStringUtils` getter methods sit behind `StringSerializer` static-final access gates. Do not add per-call access checks in those getters; missing module opens should fail at trusted-lookup initialization or cold setup, not inside string hot paths. @@ -159,10 +170,10 @@ Load this file when changing anything under `java/` or when Java drives a cross- - Do not add or restore constructor-binding APIs such as `@ForyConstructor` or `BaseFory.registerConstructor(...)`. Java parameter names, `-parameters`, and `@ConstructorProperties` are not a Fory object-creation contract. Runtime serializers for - ordinary classes must create an empty instance through `TypeResolver.getObjectCreator(Class)` and + ordinary classes must create an empty instance through `TypeResolver.getObjectInstantiator(Class)` and set fields; records and source-generated Kotlin serializers are the constructor-owned paths. - Source-generated constructor serializers must own their constructor metadata at generation time - and call constructors directly. They must not depend on runtime `ObjectCreator` constructor-field + and call constructors directly. They must not depend on runtime `ObjectInstantiator` constructor-field metadata or varargs constructor calls. - Generated JVM copy code may direct-copy immutable scalar values, but Java `Collection`/`Map` subclasses must be copied through `CopyContext.copyObject(...)` so collection/map serializers own diff --git a/integration_tests/jpms_tests/src/main/java/module-info.java b/integration_tests/jpms_tests/src/main/java/module-info.java index b769ae2bbc..fdda0aa8b6 100644 --- a/integration_tests/jpms_tests/src/main/java/module-info.java +++ b/integration_tests/jpms_tests/src/main/java/module-info.java @@ -25,7 +25,6 @@ // we can't really test any classes from this module because it only contains test-classes requires org.apache.fory.test.suite; - exports org.apache.fory.integration_tests.constructor; exports org.apache.fory.integration_tests.model; exports org.apache.fory.integration_tests.publicserializer; diff --git a/integration_tests/jpms_tests/src/main/java/org/apache/fory/integration_tests/constructor/PrivateConstructorBean.java b/integration_tests/jpms_tests/src/main/java/org/apache/fory/integration_tests/constructor/PrivateConstructorBean.java deleted file mode 100644 index 04434443f8..0000000000 --- a/integration_tests/jpms_tests/src/main/java/org/apache/fory/integration_tests/constructor/PrivateConstructorBean.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.fory.integration_tests.constructor; - -public final class PrivateConstructorBean { - private final String name; - private final int age; - - private PrivateConstructorBean(String name, int age) { - this.name = name; - this.age = age; - } - - public static PrivateConstructorBean of(String name, int age) { - return new PrivateConstructorBean(name, age); - } - - public String name() { - return name; - } - - public int age() { - return age; - } -} diff --git a/integration_tests/jpms_tests/src/test/java/org/apache/fory/integration_tests/JpmsFieldAccessorTest.java b/integration_tests/jpms_tests/src/test/java/org/apache/fory/integration_tests/JpmsFieldAccessorTest.java index ed878b1766..1381233d8d 100644 --- a/integration_tests/jpms_tests/src/test/java/org/apache/fory/integration_tests/JpmsFieldAccessorTest.java +++ b/integration_tests/jpms_tests/src/test/java/org/apache/fory/integration_tests/JpmsFieldAccessorTest.java @@ -21,7 +21,6 @@ import java.lang.reflect.Field; import org.apache.fory.Fory; -import org.apache.fory.integration_tests.constructor.PrivateConstructorBean; import org.apache.fory.integration_tests.model.PrivateFieldBean; import org.apache.fory.integration_tests.publicserializer.PublicSerializerValue; import org.apache.fory.integration_tests.publicserializer.PublicSerializerValueSerializer; @@ -89,17 +88,6 @@ public void testReflectionFinalWriteDenied() throws Exception { } } - @Test - public void testPrivateConstructorBinding() { - Fory fory = - Fory.builder().withXlang(false).withCodegen(false).requireClassRegistration(false).build(); - PrivateConstructorBean result = - (PrivateConstructorBean) - fory.deserialize(fory.serialize(PrivateConstructorBean.of("Ada", 37))); - Assert.assertEquals(result.name(), "Ada"); - Assert.assertEquals(result.age(), 37); - } - @Test public void testPublicSerializerInExportedPackage() { Fory fory = Fory.builder().withXlang(false).requireClassRegistration(false).build(); diff --git a/java/fory-annotation-processor/src/test/java/org/apache/fory/annotation/processing/ForyStructProcessorTest.java b/java/fory-annotation-processor/src/test/java/org/apache/fory/annotation/processing/ForyStructProcessorTest.java index 09a7e21d4e..c47ef56402 100644 --- a/java/fory-annotation-processor/src/test/java/org/apache/fory/annotation/processing/ForyStructProcessorTest.java +++ b/java/fory-annotation-processor/src/test/java/org/apache/fory/annotation/processing/ForyStructProcessorTest.java @@ -134,7 +134,8 @@ public void testStaticFinalFields() throws Exception { + " public int getAge() { return age; }\n" + "}\n"); Assert.assertTrue(result.success, result.diagnostics()); - String generatedSource = result.generatedSource("test/FinalFieldStruct_ForyNativeSerializer.java"); + String generatedSource = + result.generatedSource("test/FinalFieldStruct_ForyNativeSerializer.java"); Assert.assertFalse(generatedSource.contains("constructorFieldBits"), generatedSource); Assert.assertTrue(generatedSource.contains("setGeneratedFieldValue"), generatedSource); try (URLClassLoader loader = result.classLoader()) { diff --git a/java/fory-core/pom.xml b/java/fory-core/pom.xml index ed946e64ff..521af568a8 100644 --- a/java/fory-core/pom.xml +++ b/java/fory-core/pom.xml @@ -470,7 +470,7 @@ + name="META-INF/versions/25/org/apache/fory/reflect/UnsafeObjectInstantiator.java"/> @@ -487,8 +487,8 @@ file="${jdk25.sources.check.dir}/META-INF/versions/25/org/apache/fory/reflect/InstanceFieldAccessors.java" property="jdk25.instancefieldaccessors.source.present"/> + file="${jdk25.sources.check.dir}/META-INF/versions/25/org/apache/fory/reflect/UnsafeObjectInstantiator.java" + property="jdk25.unsafeobjectinstantiator.source.present"/> @@ -505,8 +505,8 @@ unless="jdk25.instancefieldaccessors.source.present" message="JDK25 versioned InstanceFieldAccessors source is missing from the source jar."/> + unless="jdk25.unsafeobjectinstantiator.source.present" + message="JDK25 versioned UnsafeObjectInstantiator source is missing from the source jar."/> diff --git a/java/fory-core/src/main/java/org/apache/fory/builder/BaseObjectCodecBuilder.java b/java/fory-core/src/main/java/org/apache/fory/builder/BaseObjectCodecBuilder.java index 4fed0ddec7..c2e670e795 100644 --- a/java/fory-core/src/main/java/org/apache/fory/builder/BaseObjectCodecBuilder.java +++ b/java/fory-core/src/main/java/org/apache/fory/builder/BaseObjectCodecBuilder.java @@ -121,7 +121,7 @@ import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.meta.TypeExtMeta; import org.apache.fory.platform.GraalvmSupport; -import org.apache.fory.reflect.ObjectCreator; +import org.apache.fory.reflect.ObjectInstantiator; import org.apache.fory.reflect.ReflectionUtils; import org.apache.fory.reflect.TypeRef; import org.apache.fory.resolver.ClassResolver; @@ -280,22 +280,22 @@ protected static T typeResolver(Fory fory, Function functio } @Override - protected void cacheObjectCreator(Class type) { - typeResolver.getObjectCreator(type); + protected void cacheObjectInstantiator(Class type) { + typeResolver.getObjectInstantiator(type); } @Override - protected Expression getObjectCreator(Class type) { - cacheObjectCreator(type); + protected Expression getObjectInstantiator(Class type) { + cacheObjectInstantiator(type); return getOrCreateField( false, - ObjectCreator.class, - ctx.newName("objectCreator_" + type.getSimpleName()), + ObjectInstantiator.class, + ctx.newName("objectInstantiator_" + type.getSimpleName()), () -> new Invoke( typeResolverRef, - "getObjectCreator", - TypeRef.of(ObjectCreator.class), + "getObjectInstantiator", + TypeRef.of(ObjectInstantiator.class), getClassExpr(type))); } diff --git a/java/fory-core/src/main/java/org/apache/fory/builder/CodecBuilder.java b/java/fory-core/src/main/java/org/apache/fory/builder/CodecBuilder.java index 40f1fedb98..46ff0aebda 100644 --- a/java/fory-core/src/main/java/org/apache/fory/builder/CodecBuilder.java +++ b/java/fory-core/src/main/java/org/apache/fory/builder/CodecBuilder.java @@ -60,8 +60,8 @@ import org.apache.fory.platform.JdkVersion; import org.apache.fory.reflect.FieldAccessor; import org.apache.fory.reflect.InstanceFieldAccessors.InstanceAccessor; -import org.apache.fory.reflect.ObjectCreator; -import org.apache.fory.reflect.ObjectCreators; +import org.apache.fory.reflect.ObjectInstantiator; +import org.apache.fory.reflect.ObjectInstantiators; import org.apache.fory.reflect.ReflectionUtils; import org.apache.fory.reflect.TypeRef; import org.apache.fory.resolver.TypeInfo; @@ -611,8 +611,9 @@ protected Expression newBean() { return new Expression.NewInstance(beanType); } else { if (JdkVersion.MAJOR_VERSION >= 25) { - cacheObjectCreator(beanClass); // trigger cache - Invoke newInstance = new Invoke(getObjectCreator(beanClass), "newInstance", OBJECT_TYPE); + cacheObjectInstantiator(beanClass); // trigger cache + Invoke newInstance = + new Invoke(getObjectInstantiator(beanClass), "newInstance", OBJECT_TYPE); return sourcePublicAccessible(beanClass) ? new Cast(newInstance, beanType) : newInstance; } Invoke newInstance = unsafeInvoke("allocateInstance", OBJECT_TYPE, beanClassExpr()); @@ -620,21 +621,21 @@ protected Expression newBean() { } } - protected void cacheObjectCreator(Class type) { - ObjectCreators.getObjectCreator(type); + protected void cacheObjectInstantiator(Class type) { + ObjectInstantiators.getObjectInstantiator(type); } - protected Expression getObjectCreator(Class type) { - cacheObjectCreator(type); + protected Expression getObjectInstantiator(Class type) { + cacheObjectInstantiator(type); return getOrCreateField( true, - ObjectCreator.class, - ctx.newName("objectCreator_" + type.getSimpleName()), + ObjectInstantiator.class, + ctx.newName("objectInstantiator_" + type.getSimpleName()), () -> new StaticInvoke( - ObjectCreators.class, - "getObjectCreator", - TypeRef.of(ObjectCreator.class), + ObjectInstantiators.class, + "getObjectInstantiator", + TypeRef.of(ObjectInstantiator.class), staticBeanClassExpr())); } diff --git a/java/fory-core/src/main/java/org/apache/fory/builder/ObjectCodecBuilder.java b/java/fory-core/src/main/java/org/apache/fory/builder/ObjectCodecBuilder.java index 026133fe8e..0395d53bc5 100644 --- a/java/fory-core/src/main/java/org/apache/fory/builder/ObjectCodecBuilder.java +++ b/java/fory-core/src/main/java/org/apache/fory/builder/ObjectCodecBuilder.java @@ -901,9 +901,10 @@ public Expression buildDecodeExpression() { FieldsCollector collector = (FieldsCollector) bean; bean = createRecord(collector.recordValuesMap); } else { - typeResolver.getObjectCreator(beanClass); // trigger cache and make error raised early + typeResolver.getObjectInstantiator(beanClass); // trigger cache and make error raised early bean = - new Invoke(getObjectCreator(beanClass), "newInstanceWithArguments", OBJECT_TYPE, bean); + new Invoke( + getObjectInstantiator(beanClass), "newInstanceWithArguments", OBJECT_TYPE, bean); } } expressions.add(new Expression.Return(bean)); diff --git a/java/fory-core/src/main/java/org/apache/fory/builder/StaticCompatibleCodecBuilder.java b/java/fory-core/src/main/java/org/apache/fory/builder/StaticCompatibleCodecBuilder.java index 26307470c5..12a3631627 100644 --- a/java/fory-core/src/main/java/org/apache/fory/builder/StaticCompatibleCodecBuilder.java +++ b/java/fory-core/src/main/java/org/apache/fory/builder/StaticCompatibleCodecBuilder.java @@ -216,7 +216,8 @@ private String genRecordCompatibleRead() { ctx.clearExprState(); Reference values = new Reference(recordValues, objectArrayTypeRef, false); Code.ExprCode newRecord = - new Invoke(getObjectCreator(beanClass), "newInstanceWithArguments", OBJECT_TYPE, values) + new Invoke( + getObjectInstantiator(beanClass), "newInstanceWithArguments", OBJECT_TYPE, values) .genCode(ctx); if (StringUtils.isNotBlank(newRecord.code())) { code.append(newRecord.code()).append('\n'); diff --git a/java/fory-core/src/main/java/org/apache/fory/codegen/Expression.java b/java/fory-core/src/main/java/org/apache/fory/codegen/Expression.java index 89090c6100..0017b83400 100644 --- a/java/fory-core/src/main/java/org/apache/fory/codegen/Expression.java +++ b/java/fory-core/src/main/java/org/apache/fory/codegen/Expression.java @@ -61,8 +61,8 @@ import java.util.stream.Stream; import org.apache.fory.builder.UnsafeCodegenSupport; import org.apache.fory.platform.JdkVersion; -import org.apache.fory.reflect.ObjectCreator; -import org.apache.fory.reflect.ObjectCreators; +import org.apache.fory.reflect.ObjectInstantiator; +import org.apache.fory.reflect.ObjectInstantiators; import org.apache.fory.reflect.ReflectionUtils; import org.apache.fory.reflect.TypeRef; import org.apache.fory.type.TypeUtils; @@ -1505,18 +1505,18 @@ public ExprCode doGenCode(CodegenContext ctx) { String functionName; String args; if (JdkVersion.MAJOR_VERSION >= 25) { - String creator = ctx.newName("objectCreator"); + String instantiator = ctx.newName("objectInstantiator"); codeBuilder .append( ExpressionUtils.callFunc( - ctx.type(ObjectCreator.class), - creator, - ctx.type(ObjectCreators.class), - "getObjectCreator", + ctx.type(ObjectInstantiator.class), + instantiator, + ctx.type(ObjectInstantiators.class), + "getObjectInstantiator", clzName + ".class", false)) .append('\n'); - target = creator; + target = instantiator; functionName = "newInstance"; args = ""; } else { diff --git a/java/fory-core/src/main/java/org/apache/fory/reflect/ObjectCreator.java b/java/fory-core/src/main/java/org/apache/fory/reflect/ObjectInstantiator.java similarity index 78% rename from java/fory-core/src/main/java/org/apache/fory/reflect/ObjectCreator.java rename to java/fory-core/src/main/java/org/apache/fory/reflect/ObjectInstantiator.java index 0335a63af0..971dcf4542 100644 --- a/java/fory-core/src/main/java/org/apache/fory/reflect/ObjectCreator.java +++ b/java/fory-core/src/main/java/org/apache/fory/reflect/ObjectInstantiator.java @@ -22,24 +22,24 @@ import javax.annotation.concurrent.ThreadSafe; /** - * Abstract base class for creating instances of a given type. + * Abstract base class for instantiating instances of a given type. * *

    This class provides a unified interface for object instantiation across different creation * strategies such as constructor invocation, unsafe allocation, and record creation. * Implementations handle various scenarios including no-arg constructors, parameterized * constructors for records, and platform-specific optimizations. * - *

    Thread Safety: All implementations of ObjectCreator are thread-safe and can - * be safely used across multiple threads concurrently. The underlying creation mechanisms + *

    Thread Safety: All implementations of ObjectInstantiator are thread-safe and + * can be safely used across multiple threads concurrently. The underlying instantiation mechanisms * (MethodHandle, Constructor, and supported constructor-bypassing allocation) are all thread-safe. * - * @param the type of objects this creator can instantiate + * @param the type of objects this instantiator can instantiate */ @ThreadSafe -public abstract class ObjectCreator { +public abstract class ObjectInstantiator { protected final Class type; - protected ObjectCreator(Class type) { + protected ObjectInstantiator(Class type) { this.type = type; } @@ -48,7 +48,8 @@ protected ObjectCreator(Class type) { * * @return a new instance of type T * @throws RuntimeException if instance creation fails - * @throws UnsupportedOperationException if this creator doesn't support parameterless creation + * @throws UnsupportedOperationException if this instantiator doesn't support parameterless + * creation */ public abstract T newInstance(); @@ -61,7 +62,8 @@ protected ObjectCreator(Class type) { * @param arguments the arguments to pass to the constructor * @return a new instance of type T * @throws RuntimeException if instance creation fails - * @throws UnsupportedOperationException if this creator doesn't support parameterized creation + * @throws UnsupportedOperationException if this instantiator doesn't support parameterized + * creation */ public abstract T newInstanceWithArguments(Object... arguments); } diff --git a/java/fory-core/src/main/java/org/apache/fory/reflect/ObjectCreators.java b/java/fory-core/src/main/java/org/apache/fory/reflect/ObjectInstantiators.java similarity index 64% rename from java/fory-core/src/main/java/org/apache/fory/reflect/ObjectCreators.java rename to java/fory-core/src/main/java/org/apache/fory/reflect/ObjectInstantiators.java index 930e53fd70..aae9484e88 100644 --- a/java/fory-core/src/main/java/org/apache/fory/reflect/ObjectCreators.java +++ b/java/fory-core/src/main/java/org/apache/fory/reflect/ObjectInstantiators.java @@ -21,9 +21,10 @@ import java.io.Serializable; import java.lang.invoke.MethodHandle; +import java.lang.invoke.MethodHandles; +import java.lang.invoke.MethodType; import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; import java.lang.reflect.Modifier; import org.apache.fory.annotation.Internal; import org.apache.fory.collection.ClassValueCache; @@ -32,98 +33,106 @@ import org.apache.fory.platform.AndroidSupport; import org.apache.fory.platform.GraalvmSupport; import org.apache.fory.platform.JdkVersion; +import org.apache.fory.platform.internal._JDKAccess; import org.apache.fory.util.record.RecordUtils; /** - * Factory class for creating {@link ObjectCreator} instances. + * Factory class for creating {@link ObjectInstantiator} instances. * - *

    This class provides a centralized way to obtain optimized object creators for different types. - * It automatically selects the most appropriate creation strategy based on the target type and - * runtime environment: + *

    This class provides a centralized way to obtain optimized object instantiators for different + * types. It automatically selects the most appropriate creation strategy based on the target type + * and runtime environment: * *

      - *
    • Record types: Uses {@link RecordObjectCreator} with MethodHandle for + *
    • Record types: Uses {@link RecordObjectInstantiator} with MethodHandle for * parameterized constructor invocation *
    • Classes with no-arg constructors: Uses {@link - * DeclaredNoArgCtrObjectCreator} with MethodHandle for fast invocation - *
    • Classes without accessible constructors: Uses a private - * constructor-bypassing creator on runtimes where that is still supported + * DeclaredNoArgCtrInstantiator} with MethodHandle for fast invocation + *
    • Classes without accessible constructors: Uses JDK8-24 Unsafe allocation or + * serialization constructor creation through the runtime ReflectionFactory owner *
    • Android compatibility: Uses reflection for records and no-arg * constructors, and throws when no supported reflective construction path exists *
    * - *

    The static {@link #getObjectCreator(Class)} method keeps the legacy process-global cache. - * Runtime-owned paths should use {@link - * org.apache.fory.resolver.TypeResolver#getObjectCreator(Class)} so ObjectStream-compatible - * creators stay scoped to the Fory runtime. + *

    The static {@link #getObjectInstantiator(Class)} method keeps the process-global construction + * cache. Runtime-owned paths should use {@link + * org.apache.fory.resolver.TypeResolver#getObjectInstantiator(Class)} so ObjectStream-compatible + * instantiators stay scoped to the Fory runtime. * - *

    Thread Safety: This class and all returned ObjectCreator instances are + *

    Thread Safety: This class and all returned ObjectInstantiator instances are * thread-safe and can be safely used across multiple threads concurrently. */ @SuppressWarnings("unchecked") -public class ObjectCreators { - private static final ClassValueCache> cache = +public class ObjectInstantiators { + private static final ClassValueCache> cache = ClassValueCache.newClassKeySoftCache(8); /** - * Returns an optimized ObjectCreator for the given type. + * Returns an optimized ObjectInstantiator for the given type. * *

    This method automatically selects the most appropriate creation strategy based on the type * characteristics and caches the result for future use. The selection logic prioritizes * performance and platform compatibility. * - * @param the type for which to create an ObjectCreator + * @param the type for which to create an ObjectInstantiator * @param type the Class object representing the target type - * @return a cached ObjectCreator instance optimized for the given type + * @return a cached ObjectInstantiator instance optimized for the given type * @throws ForyException if the type cannot be instantiated (e.g., missing no-arg constructor in * GraalVM native image) */ - public static ObjectCreator getObjectCreator(Class type) { - return (ObjectCreator) cache.get(type, () -> createObjectCreator(type)); + public static ObjectInstantiator getObjectInstantiator(Class type) { + return (ObjectInstantiator) cache.get(type, () -> createObjectInstantiator(type)); } - /** Creates an uncached object creator for runtime-scoped registries. */ + /** Creates an uncached object instantiator for runtime-scoped registries. */ @Internal - public static ObjectCreator createObjectCreator(Class type) { + public static ObjectInstantiator createObjectInstantiator(Class type) { if (RecordUtils.isRecord(type)) { - return new RecordObjectCreator<>(type); + return new RecordObjectInstantiator<>(type); } Constructor noArgConstructor = ReflectionUtils.getNoArgConstructor(type); if (AndroidSupport.IS_ANDROID) { if (noArgConstructor != null) { - return new ReflectiveNoArgCtrObjectCreator<>(type, noArgConstructor); + return new ReflectiveNoArgCtrInstantiator<>(type, noArgConstructor); } - return new UnsupportedObjectCreator<>( + return new UnsupportedObjectInstantiator<>( type, "Android cannot create " + type + " without an accessible no-arg constructor"); } if (GraalvmSupport.IN_GRAALVM_NATIVE_IMAGE) { if (noArgConstructor != null) { - return new DeclaredNoArgCtrObjectCreator<>(type); + return new DeclaredNoArgCtrInstantiator<>(type); + } else if (JdkVersion.MAJOR_VERSION >= 25) { + return new ParentNoArgCtrInstantiator<>(type); } else { - return new ConstructorBypassObjectCreator<>(type); + return new UnsafeObjectInstantiator<>(type); } } if (noArgConstructor == null) { - return new ConstructorBypassObjectCreator<>(type); + if (JdkVersion.MAJOR_VERSION >= 25) { + return new ParentNoArgCtrInstantiator<>(type); + } + return new UnsafeObjectInstantiator<>(type); } - return new DeclaredNoArgCtrObjectCreator<>(type); + return new DeclaredNoArgCtrInstantiator<>(type); } - /** Creates an uncached empty-instance creator for Java ObjectStream-compatible serializers. */ + /** + * Creates an uncached empty-instance instantiator for Java ObjectStream-compatible serializers. + */ @Internal - public static ObjectCreator createObjectStreamCreator(Class type) { + public static ObjectInstantiator createObjectStreamInstantiator(Class type) { if (AndroidSupport.IS_ANDROID) { Constructor noArgConstructor = ReflectionUtils.getNoArgConstructor(type); if (noArgConstructor != null) { - return new ReflectiveNoArgCtrObjectCreator<>(type, noArgConstructor); + return new ReflectiveNoArgCtrInstantiator<>(type, noArgConstructor); } - return new UnsupportedObjectCreator<>( + return new UnsupportedObjectInstantiator<>( type, "Android cannot create " + type + " without an accessible no-arg constructor"); } - if (GraalvmSupport.IN_GRAALVM_NATIVE_IMAGE) { - return new ConstructorBypassObjectCreator<>(type); + if (GraalvmSupport.IN_GRAALVM_NATIVE_IMAGE && JdkVersion.MAJOR_VERSION < 25) { + return new UnsafeObjectInstantiator<>(type); } - return new ParentNoArgCtrObjectCreator<>(type); + return new ParentNoArgCtrInstantiator<>(type); } private static RuntimeException makeException(Class type, Throwable cause) { @@ -143,10 +152,10 @@ private static Throwable unwrapConstructorFailure(Throwable cause) { return cause; } - private static final class ReflectiveNoArgCtrObjectCreator extends ObjectCreator { + private static final class ReflectiveNoArgCtrInstantiator extends ObjectInstantiator { private final Constructor constructor; - private ReflectiveNoArgCtrObjectCreator(Class type, Constructor constructor) { + private ReflectiveNoArgCtrInstantiator(Class type, Constructor constructor) { super(type); this.constructor = constructor; try { @@ -171,10 +180,10 @@ public T newInstanceWithArguments(Object... arguments) { } } - private static final class UnsupportedObjectCreator extends ObjectCreator { + private static final class UnsupportedObjectInstantiator extends ObjectInstantiator { private final String message; - private UnsupportedObjectCreator(Class type, String message) { + private UnsupportedObjectInstantiator(Class type, String message) { super(type); this.message = message; } @@ -190,29 +199,10 @@ public T newInstanceWithArguments(Object... arguments) { } } - private static final class ConstructorBypassObjectCreator extends ObjectCreator { - private final ConstructorBypassAllocator allocator; - - public ConstructorBypassObjectCreator(Class type) { - super(type); - allocator = new ConstructorBypassAllocator<>(type); - } - - @Override - public T newInstance() { - return allocator.allocate(); - } - - @Override - public T newInstanceWithArguments(Object... arguments) { - throw new UnsupportedOperationException(); - } - } - - public static final class DeclaredNoArgCtrObjectCreator extends ObjectCreator { + public static final class DeclaredNoArgCtrInstantiator extends ObjectInstantiator { private final MethodHandle handle; - public DeclaredNoArgCtrObjectCreator(Class type) { + public DeclaredNoArgCtrInstantiator(Class type) { super(type); handle = ReflectionUtils.getCtrHandle(type, true); } @@ -232,11 +222,11 @@ public T newInstanceWithArguments(Object... arguments) { } } - public static final class RecordObjectCreator extends ObjectCreator { + public static final class RecordObjectInstantiator extends ObjectInstantiator { private final MethodHandle handle; private final Constructor constructor; - public RecordObjectCreator(Class type) { + public RecordObjectInstantiator(Class type) { super(type); Tuple2 tuple2 = RecordUtils.getRecordConstructor(type); constructor = tuple2.f0; @@ -278,37 +268,19 @@ public T newInstanceWithArguments(Object... arguments) { } } - public static final class ParentNoArgCtrObjectCreator extends ObjectCreator { - private static volatile Object reflectionFactory; - private static volatile Method newConstructorForSerializationMethod; - + public static final class ParentNoArgCtrInstantiator extends ObjectInstantiator { private final Constructor constructor; - private final ConstructorBypassAllocator allocator; - public ParentNoArgCtrObjectCreator(Class type) { + public ParentNoArgCtrInstantiator(Class type) { super(type); - if (JdkVersion.MAJOR_VERSION >= 25) { - constructor = null; - allocator = new ConstructorBypassAllocator<>(type); - return; - } this.constructor = createSerializationConstructor(type); - allocator = null; } private static Constructor createSerializationConstructor(Class type) { try { - if (reflectionFactory == null) { - Class reflectionFactoryClass = Class.forName("sun.reflect.ReflectionFactory"); - Method getReflectionFactory = reflectionFactoryClass.getMethod("getReflectionFactory"); - reflectionFactory = getReflectionFactory.invoke(null); - newConstructorForSerializationMethod = - reflectionFactoryClass.getMethod( - "newConstructorForSerialization", Class.class, Constructor.class); - } Constructor parentConstructor = findSerializationConstructor(type); return (Constructor) - newConstructorForSerializationMethod.invoke(reflectionFactory, type, parentConstructor); + ReflectionFactoryAccess.newConstructorForSerialization(type, parentConstructor); } catch (Throwable e) { throw new ForyException( "Failed to create instance, please provide a no-arg constructor for " + type, e); @@ -351,10 +323,6 @@ private static boolean validSerializationConstructor( @Override public T newInstance() { - ConstructorBypassAllocator constructorBypassAllocator = allocator; - if (constructorBypassAllocator != null) { - return constructorBypassAllocator.allocate(); - } try { return constructor.newInstance(); } catch (Exception e) { @@ -366,5 +334,43 @@ public T newInstance() { public T newInstanceWithArguments(Object... arguments) { throw new UnsupportedOperationException(); } + + private static final class ReflectionFactoryAccess { + private static final Object REFLECTION_FACTORY; + private static final MethodHandle NEW_CONSTRUCTOR_FOR_SERIALIZATION; + + static { + try { + Class reflectionFactoryClass = reflectionFactoryClass(); + MethodHandles.Lookup lookup = _JDKAccess._trustedLookup(reflectionFactoryClass); + MethodHandle getReflectionFactory = + lookup.findStatic( + reflectionFactoryClass, + "getReflectionFactory", + MethodType.methodType(reflectionFactoryClass)); + REFLECTION_FACTORY = getReflectionFactory.invoke(); + NEW_CONSTRUCTOR_FOR_SERIALIZATION = + lookup.findVirtual( + reflectionFactoryClass, + "newConstructorForSerialization", + MethodType.methodType(Constructor.class, Class.class, Constructor.class)); + } catch (Throwable e) { + throw new ExceptionInInitializerError(e); + } + } + + private static Class reflectionFactoryClass() throws ClassNotFoundException { + if (JdkVersion.MAJOR_VERSION >= 25) { + return Class.forName("jdk.internal.reflect.ReflectionFactory"); + } + return Class.forName("sun.reflect.ReflectionFactory"); + } + + private static Constructor newConstructorForSerialization( + Class type, Constructor parentConstructor) throws Throwable { + return (Constructor) + NEW_CONSTRUCTOR_FOR_SERIALIZATION.invoke(REFLECTION_FACTORY, type, parentConstructor); + } + } } } diff --git a/java/fory-core/src/main/java/org/apache/fory/reflect/ConstructorBypassAllocator.java b/java/fory-core/src/main/java/org/apache/fory/reflect/UnsafeObjectInstantiator.java similarity index 82% rename from java/fory-core/src/main/java/org/apache/fory/reflect/ConstructorBypassAllocator.java rename to java/fory-core/src/main/java/org/apache/fory/reflect/UnsafeObjectInstantiator.java index 9cb859c63d..85503d32b2 100644 --- a/java/fory-core/src/main/java/org/apache/fory/reflect/ConstructorBypassAllocator.java +++ b/java/fory-core/src/main/java/org/apache/fory/reflect/UnsafeObjectInstantiator.java @@ -26,20 +26,20 @@ import org.apache.fory.platform.internal._UnsafeUtils; import sun.misc.Unsafe; -/** Internal JDK8-24 constructor-bypass allocator used by object creators. */ +/** JDK8-24 Unsafe-backed instantiator for classes without an invocable constructor. */ +@SuppressWarnings("unchecked") @Internal -final class ConstructorBypassAllocator { +final class UnsafeObjectInstantiator extends ObjectInstantiator { private static final Unsafe UNSAFE = AndroidSupport.IS_ANDROID ? null : _UnsafeUtils.UNSAFE; private static final boolean UNSAFE_ALLOCATION_AVAILABLE = UNSAFE != null && JdkVersion.MAJOR_VERSION < 25; - private final Class type; - - ConstructorBypassAllocator(Class type) { - this.type = type; + UnsafeObjectInstantiator(Class type) { + super(type); } - T allocate() { + @Override + public T newInstance() { if (!UNSAFE_ALLOCATION_AVAILABLE) { throw unsupported(type); } @@ -50,6 +50,11 @@ T allocate() { } } + @Override + public T newInstanceWithArguments(Object... arguments) { + throw new UnsupportedOperationException(); + } + private static ForyException unsupported(Class type) { return new ForyException( "Constructor-bypassing Unsafe allocation is unsupported in this runtime for " + type); diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java index 59515e120a..1159216721 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java @@ -1867,12 +1867,12 @@ private void registerGraalvmSerializerClass(Class cls) { RecordUtils.getRecordConstructor(cls); RecordUtils.getRecordComponents(cls); } - if (needsGraalvmObjectCreator(cls, serializerClass)) { - getObjectCreator(cls); + if (needsGraalvmObjectInstantiator(cls, serializerClass)) { + getObjectInstantiator(cls); } } - private boolean needsGraalvmObjectCreator( + private boolean needsGraalvmObjectInstantiator( Class cls, Class serializerClass) { if (cls.isArray()) { return false; diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/SharedRegistry.java b/java/fory-core/src/main/java/org/apache/fory/resolver/SharedRegistry.java index c7311ea78b..a3f573afdd 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/SharedRegistry.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/SharedRegistry.java @@ -39,8 +39,8 @@ import org.apache.fory.meta.MetaStringEncoder; import org.apache.fory.meta.TypeDef; import org.apache.fory.platform.GraalvmSupport; -import org.apache.fory.reflect.ObjectCreator; -import org.apache.fory.reflect.ObjectCreators; +import org.apache.fory.reflect.ObjectInstantiator; +import org.apache.fory.reflect.ObjectInstantiators; import org.apache.fory.serializer.Serializer; import org.apache.fory.type.Descriptor; import org.apache.fory.type.DescriptorGrouper; @@ -83,9 +83,9 @@ public final class SharedRegistry { new ConcurrentIdentityMap<>(); final ConcurrentIdentityMap, Serializer> registeredSerializerCache = new ConcurrentIdentityMap<>(); - private final ConcurrentHashMap, ObjectCreator> objectCreatorCache = + private final ConcurrentHashMap, ObjectInstantiator> objectInstantiatorCache = new ConcurrentHashMap<>(); - private final ConcurrentHashMap, ObjectCreator> objectStreamCreatorCache = + private final ConcurrentHashMap, ObjectInstantiator> objectStreamInstantiatorCache = new ConcurrentHashMap<>(); final StaticGeneratedSerializerRegistry staticGeneratedSerializerRegistry = new StaticGeneratedSerializerRegistry(); @@ -132,22 +132,18 @@ Serializer cacheRegisteredSerializer(Class type, Serializer serializer) } @SuppressWarnings("unchecked") - public ObjectCreator getObjectCreator(Class type) { - return (ObjectCreator) - objectCreatorCache.computeIfAbsent(type, ObjectCreators::createObjectCreator); + public ObjectInstantiator getObjectInstantiator(Class type) { + return (ObjectInstantiator) + objectInstantiatorCache.computeIfAbsent( + type, ObjectInstantiators::createObjectInstantiator); } - /** - * Returns the runtime-scoped creator used by Java ObjectStream-compatible serializers. - * - *

    ObjectStream reconstruction creates an empty instance before stream fields are read, so - * explicit constructor mappings registered for normal object serializers are not semantically - * valid for this path. - */ + /** Returns the runtime-scoped instantiator used by Java ObjectStream-compatible serializers. */ @SuppressWarnings("unchecked") - public ObjectCreator getObjectStreamCreator(Class type) { - return (ObjectCreator) - objectStreamCreatorCache.computeIfAbsent(type, ObjectCreators::createObjectStreamCreator); + public ObjectInstantiator getObjectStreamInstantiator(Class type) { + return (ObjectInstantiator) + objectStreamInstantiatorCache.computeIfAbsent( + type, ObjectInstantiators::createObjectStreamInstantiator); } TypeInfo cacheRegisteredTypeInfo(Class type, TypeInfo typeInfo) { diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java index 4ff3817624..a1a4377a27 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java @@ -77,7 +77,7 @@ import org.apache.fory.meta.TypeExtMeta; import org.apache.fory.platform.AndroidSupport; import org.apache.fory.platform.GraalvmSupport; -import org.apache.fory.reflect.ObjectCreator; +import org.apache.fory.reflect.ObjectInstantiator; import org.apache.fory.reflect.ReflectionUtils; import org.apache.fory.reflect.TypeRef; import org.apache.fory.serializer.CodegenSerializer; @@ -343,14 +343,15 @@ public abstract void registerEnum( Class type, String namespace, String typeName, Serializer serializer); /** - * Returns the runtime-scoped object creator for {@code type}. + * Returns the runtime-scoped object instantiator for {@code type}. * - *

    The creator respects constructor mappings registered through {@code Fory} and annotations on - * the target type. + *

    The instantiator follows the normal Java object model used by Fory serializers. Records use + * their canonical constructor, while ordinary classes use supported empty construction followed + * by field restoration. */ @Internal - public final ObjectCreator getObjectCreator(Class type) { - return sharedRegistry.getObjectCreator(type); + public final ObjectInstantiator getObjectInstantiator(Class type) { + return sharedRegistry.getObjectInstantiator(type); } /** diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/AbstractObjectSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/AbstractObjectSerializer.java index 2ffac30bf1..1bfd2a5e7c 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/AbstractObjectSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/AbstractObjectSerializer.java @@ -43,7 +43,7 @@ import org.apache.fory.logging.LoggerFactory; import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.reflect.FieldAccessor; -import org.apache.fory.reflect.ObjectCreator; +import org.apache.fory.reflect.ObjectInstantiator; import org.apache.fory.reflect.ReflectionUtils; import org.apache.fory.reflect.TypeRef; import org.apache.fory.resolver.RefMode; @@ -78,7 +78,7 @@ public abstract class AbstractObjectSerializer extends Serializer { protected final Config config; protected final TypeResolver typeResolver; protected final boolean isRecord; - protected final ObjectCreator objectCreator; + protected final ObjectInstantiator objectInstantiator; private SerializationFieldInfo[] fieldInfos; private RecordInfo copyRecordInfo; @@ -87,20 +87,20 @@ protected AbstractObjectSerializer() { this.config = null; this.typeResolver = null; this.isRecord = false; - this.objectCreator = null; + this.objectInstantiator = null; } public AbstractObjectSerializer(TypeResolver typeResolver, Class type) { - this(typeResolver, type, typeResolver.getObjectCreator(type)); + this(typeResolver, type, typeResolver.getObjectInstantiator(type)); } public AbstractObjectSerializer( - TypeResolver typeResolver, Class type, ObjectCreator objectCreator) { + TypeResolver typeResolver, Class type, ObjectInstantiator objectInstantiator) { super(typeResolver.getConfig(), type); this.config = typeResolver.getConfig(); this.typeResolver = typeResolver; this.isRecord = RecordUtils.isRecord(type); - this.objectCreator = objectCreator; + this.objectInstantiator = objectInstantiator; } static void writeField( @@ -506,8 +506,7 @@ static Object readContainerFieldValue( break; case TRACKING: generics.pushGenericType(fieldInfo.genericType, readContext.getDepth()); - fieldValue = - readContainerFieldValueRef(readContext, typeResolver, refReader, fieldInfo); + fieldValue = readContainerFieldValueRef(readContext, typeResolver, refReader, fieldInfo); generics.popGenericType(readContext.getDepth()); break; default: @@ -885,7 +884,7 @@ private T copyRecord(CopyContext copyContext, T originObj) { Object[] fieldValues = copyFieldValues(copyContext, originObj); fieldValues = RecordUtils.remapping(copyRecordInfo, fieldValues); try { - T t = objectCreator.newInstanceWithArguments(fieldValues); + T t = objectInstantiator.newInstanceWithArguments(fieldValues); Arrays.fill(copyRecordInfo.getRecordComponents(), null); copyContext.reference(originObj, t); return t; @@ -1084,7 +1083,7 @@ private SerializationFieldInfo[] buildFieldsInfo() { } protected T newBean() { - return objectCreator.newInstance(); + return objectInstantiator.newInstance(); } protected final void checkNoUnresolvedReadRef(ReadContext readContext) { diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/CompatibleLayerSerializerBase.java b/java/fory-core/src/main/java/org/apache/fory/serializer/CompatibleLayerSerializerBase.java index a93a2f250f..b696a21a67 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/CompatibleLayerSerializerBase.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/CompatibleLayerSerializerBase.java @@ -30,7 +30,7 @@ import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.meta.TypeDef; import org.apache.fory.reflect.FieldAccessor; -import org.apache.fory.reflect.ObjectCreator; +import org.apache.fory.reflect.ObjectInstantiator; import org.apache.fory.resolver.TypeResolver; import org.apache.fory.serializer.FieldGroups.SerializationFieldInfo; import org.apache.fory.type.DescriptorGrouper; @@ -43,10 +43,13 @@ */ @SuppressWarnings({"unchecked", "rawtypes"}) public abstract class CompatibleLayerSerializerBase extends AbstractObjectSerializer { - private static final ObjectCreator FIELD_ONLY_CREATOR = new FieldOnlyCreator(); + // Layer serializers are field-only views over an already-created subclass instance. A real + // object instantiator would move construction ownership into each superclass slot serializer. + private static final ObjectInstantiator FIELD_ONLY_INSTANTIATOR = + new FieldOnlyInstantiator(); - private static final class FieldOnlyCreator extends ObjectCreator { - private FieldOnlyCreator() { + private static final class FieldOnlyInstantiator extends ObjectInstantiator { + private FieldOnlyInstantiator() { super(Object.class); } @@ -66,11 +69,11 @@ public Object newInstanceWithArguments(Object... arguments) { protected SerializationFieldInfo[] allFields = new SerializationFieldInfo[0]; public CompatibleLayerSerializerBase(TypeResolver typeResolver, Class type) { - super(typeResolver, type, fieldOnlyCreator()); + super(typeResolver, type, fieldOnlyInstantiator()); } - private static ObjectCreator fieldOnlyCreator() { - return (ObjectCreator) FIELD_ONLY_CREATOR; + private static ObjectInstantiator fieldOnlyInstantiator() { + return (ObjectInstantiator) FIELD_ONLY_INSTANTIATOR; } public final void setLayerSerializerMeta(TypeDef layerTypeDef, Class layerMarkerClass) { diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/CompatibleSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/CompatibleSerializer.java index ca88cebde3..4d27fe869f 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/CompatibleSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/CompatibleSerializer.java @@ -244,7 +244,7 @@ public T read(ReadContext readContext) { readFields(readContext, fieldValues); } fieldValues = RecordUtils.remapping(recordInfo, fieldValues); - T t = objectCreator.newInstanceWithArguments(fieldValues); + T t = objectInstantiator.newInstanceWithArguments(fieldValues); Arrays.fill(recordInfo.getRecordComponents(), null); return t; } diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/ExceptionSerializers.java b/java/fory-core/src/main/java/org/apache/fory/serializer/ExceptionSerializers.java index 1fba01ea5b..c13b52cc85 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/ExceptionSerializers.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/ExceptionSerializers.java @@ -45,8 +45,8 @@ import org.apache.fory.platform.JdkVersion; import org.apache.fory.platform.internal._JDKAccess; import org.apache.fory.reflect.FieldAccessor; -import org.apache.fory.reflect.ObjectCreator; -import org.apache.fory.reflect.ObjectCreators; +import org.apache.fory.reflect.ObjectInstantiator; +import org.apache.fory.reflect.ObjectInstantiators; import org.apache.fory.reflect.ReflectionUtils; import org.apache.fory.resolver.TypeResolver; @@ -54,12 +54,15 @@ @SuppressWarnings({"rawtypes", "unchecked"}) public final class ExceptionSerializers { private static final Set> THROWABLE_SUPER_CLASSES = ofHashSet(Throwable.class); - private static final ObjectCreator FIELD_ONLY_CREATOR = new FieldOnlyCreator(); + // Throwable slot serializers populate fields on the throwable allocated by ExceptionSerializer. + // They must not resolve constructors for each serialized superclass layer. + private static final ObjectInstantiator FIELD_ONLY_INSTANTIATOR = + new FieldOnlyInstantiator(); private ExceptionSerializers() {} - private static final class FieldOnlyCreator extends ObjectCreator { - private FieldOnlyCreator() { + private static final class FieldOnlyInstantiator extends ObjectInstantiator { + private FieldOnlyInstantiator() { super(Object.class); } @@ -77,7 +80,7 @@ public Object newInstanceWithArguments(Object... arguments) { public static final class ExceptionSerializer extends Serializer { private final Config config; private final TypeResolver typeResolver; - private final ObjectCreator objectCreator; + private final ObjectInstantiator objectInstantiator; private final Constructor messageConstructor; private volatile Serializer[] slotsSerializers; private volatile boolean rebuildSlotsSerializersAtRuntime; @@ -87,9 +90,9 @@ public ExceptionSerializer(TypeResolver typeResolver, Class type) { this.config = typeResolver.getConfig(); this.typeResolver = typeResolver; messageConstructor = getOptionalMessageConstructor(type); - objectCreator = + objectInstantiator = messageConstructor == null && MemoryUtils.JDK_LANG_FIELD_ACCESS - ? createThrowableObjectCreator(typeResolver, type) + ? createThrowableObjectInstantiator(typeResolver, type) : null; slotsSerializers = buildSlotsSerializers(typeResolver, type); if (!MemoryUtils.JDK_LANG_FIELD_ACCESS @@ -198,7 +201,7 @@ private T newThrowableForRead() { + " without a String message constructor because it requires Unsafe allocation " + "or unsupported private-field access."); } - return objectCreator.newInstance(); + return objectInstantiator.newInstance(); } private T newThrowableWithMessage(String detailMessage) { @@ -380,15 +383,15 @@ private static StackTraceElement newStackTraceElement( } } - private static ObjectCreator createThrowableObjectCreator( + private static ObjectInstantiator createThrowableObjectInstantiator( TypeResolver typeResolver, Class type) { if (GraalvmSupport.IN_GRAALVM_NATIVE_IMAGE || JdkVersion.MAJOR_VERSION >= 25) { - return typeResolver.getObjectCreator(type); + return typeResolver.getObjectInstantiator(type); } if (ReflectionUtils.getCtrHandle(type, false) != null) { - return typeResolver.getObjectCreator(type); + return typeResolver.getObjectInstantiator(type); } - return new ObjectCreators.ParentNoArgCtrObjectCreator<>(type); + return new ObjectInstantiators.ParentNoArgCtrInstantiator<>(type); } private static Constructor getOptionalMessageConstructor(Class type) { @@ -441,7 +444,8 @@ private static Serializer[] buildSlotsSerializers(TypeResolver typeResolver, slotsSerializer = new CompatibleLayerSerializer(typeResolver, type, layerTypeDef, layerMarkerClass); } else { - slotsSerializer = new ObjectSerializer<>(typeResolver, type, false, fieldOnlyCreator()); + slotsSerializer = + new ObjectSerializer<>(typeResolver, type, false, fieldOnlyInstantiator()); } serializers.add(slotsSerializer); type = (Class) type.getSuperclass(); @@ -451,8 +455,8 @@ private static Serializer[] buildSlotsSerializers(TypeResolver typeResolver, return serializers.toArray(new Serializer[0]); } - private static ObjectCreator fieldOnlyCreator() { - return (ObjectCreator) FIELD_ONLY_CREATOR; + private static ObjectInstantiator fieldOnlyInstantiator() { + return (ObjectInstantiator) FIELD_ONLY_INSTANTIATOR; } private static void readAndSetFields( diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/ExternalizableSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/ExternalizableSerializer.java index f1539d83bf..e9cd3a92ff 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/ExternalizableSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/ExternalizableSerializer.java @@ -57,7 +57,7 @@ public void write(WriteContext writeContext, T value) { @Override public T read(ReadContext readContext) { - T t = objectCreator.newInstance(); + T t = objectInstantiator.newInstance(); readContext.reference(t); objectInput.setReadContext(readContext); try { diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectSerializer.java index 1ef5f1fbfb..e656359d14 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectSerializer.java @@ -33,7 +33,7 @@ import org.apache.fory.logging.LoggerFactory; import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.meta.TypeDef; -import org.apache.fory.reflect.ObjectCreator; +import org.apache.fory.reflect.ObjectInstantiator; import org.apache.fory.resolver.TypeResolver; import org.apache.fory.serializer.FieldGroups.SerializationFieldInfo; import org.apache.fory.serializer.struct.Fingerprint; @@ -71,15 +71,15 @@ public ObjectSerializer(TypeResolver typeResolver, Class cls) { } public ObjectSerializer(TypeResolver typeResolver, Class cls, boolean resolveParent) { - this(typeResolver, cls, resolveParent, typeResolver.getObjectCreator(cls)); + this(typeResolver, cls, resolveParent, typeResolver.getObjectInstantiator(cls)); } public ObjectSerializer( TypeResolver typeResolver, Class cls, boolean resolveParent, - ObjectCreator objectCreator) { - super(typeResolver, cls, objectCreator); + ObjectInstantiator objectInstantiator) { + super(typeResolver, cls, objectInstantiator); // avoid recursive building serializers. // Use `setSerializerIfAbsent` to avoid overwriting existing serializer for class when used // as data serializer. @@ -212,7 +212,7 @@ public T read(ReadContext readContext) { if (isRecord) { Object[] fields = readFields(readContext); fields = RecordUtils.remapping(recordInfo, fields); - T obj = objectCreator.newInstanceWithArguments(fields); + T obj = objectInstantiator.newInstanceWithArguments(fields); Arrays.fill(recordInfo.getRecordComponents(), null); return obj; } diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectStreamSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectStreamSerializer.java index 2075a193c3..20a3851e1e 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectStreamSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectStreamSerializer.java @@ -162,7 +162,7 @@ CompatibleLayerSerializerBase getReadSerializer( * Safe wrapper for ObjectStreamClass.lookup that handles GraalVM native image limitations. In * GraalVM native image, ObjectStreamClass.lookup may fail for certain classes like Throwable due * to missing SerializationConstructorAccessor. This method catches such errors and returns null - * so the serializer can use its constructor-bypassing object creator. + * so the serializer can use its constructor-bypassing object instantiator. */ private static ObjectStreamClass safeObjectStreamClassLookup(Class type) { if (GraalvmSupport.IN_GRAALVM_NATIVE_IMAGE) { @@ -171,7 +171,7 @@ private static ObjectStreamClass safeObjectStreamClassLookup(Class type) { } catch (Throwable e) { // In GraalVM native image, ObjectStreamClass.lookup may fail for certain classes due to // missing SerializationConstructorAccessor. Returning null keeps stream reconstruction on - // the serializer-owned object creator path. + // the serializer-owned object instantiator path. LOG.warn( "ObjectStreamClass.lookup failed for {} in GraalVM native image: {}", type.getName(), @@ -185,7 +185,7 @@ private static ObjectStreamClass safeObjectStreamClassLookup(Class type) { } public ObjectStreamSerializer(TypeResolver typeResolver, Class type) { - super(typeResolver, type, typeResolver.getSharedRegistry().getObjectStreamCreator(type)); + super(typeResolver, type, typeResolver.getSharedRegistry().getObjectStreamInstantiator(type)); if (!Serializable.class.isAssignableFrom(type)) { throw new IllegalArgumentException( String.format("Class %s should implement %s.", type, Serializable.class)); @@ -269,7 +269,7 @@ public void write(WriteContext writeContext, Object value) { @Override public Object read(ReadContext readContext) { MemoryBuffer buffer = readContext.getBuffer(); - Object obj = objectCreator.newInstance(); + Object obj = objectInstantiator.newInstance(); readContext.reference(obj); int numClasses = buffer.readInt16(); int slotIndex = 0; @@ -420,7 +420,7 @@ public Object copy(CopyContext copyContext, Object value) { if (!canCopyWithDefaultReadObject()) { return super.copy(copyContext, value); } - Object copy = objectCreator.newInstance(); + Object copy = objectInstantiator.newInstance(); copyContext.reference(value, copy); try { for (SlotInfo slotInfo : slotsInfos) { diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/SerializationHookLookup.java b/java/fory-core/src/main/java/org/apache/fory/serializer/SerializationHookLookup.java index 6cbfe2e1e2..a07ee825d1 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/SerializationHookLookup.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/SerializationHookLookup.java @@ -23,6 +23,7 @@ import java.io.ObjectOutputStream; import java.lang.invoke.MethodHandle; import java.lang.invoke.MethodHandles; +import java.lang.invoke.MethodType; import java.lang.reflect.Method; import java.lang.reflect.Modifier; import org.apache.fory.collection.ClassValueCache; @@ -40,39 +41,42 @@ static MethodHandle readResolveHandle(Class type, Method method) private static final class Methods { private static final Object REFLECTION_FACTORY; - private static final Method WRITE_OBJECT; - private static final Method READ_OBJECT; - private static final Method READ_OBJECT_NO_DATA; - private static final Method DEFAULT_READ_OBJECT; - private static final Method WRITE_REPLACE; - private static final Method READ_RESOLVE; + private static final MethodHandle WRITE_OBJECT; + private static final MethodHandle READ_OBJECT; + private static final MethodHandle READ_OBJECT_NO_DATA; + private static final MethodHandle DEFAULT_READ_OBJECT; + private static final MethodHandle WRITE_REPLACE; + private static final MethodHandle READ_RESOLVE; static { Object reflectionFactory = null; - Method writeObject = null; - Method readObject = null; - Method readObjectNoData = null; - Method defaultReadObject = null; - Method writeReplace = null; - Method readResolve = null; + MethodHandle writeObject = null; + MethodHandle readObject = null; + MethodHandle readObjectNoData = null; + MethodHandle defaultReadObject = null; + MethodHandle writeReplace = null; + MethodHandle readResolve = null; if (JdkVersion.MAJOR_VERSION < 25) { try { Class factoryClass = Class.forName("sun.reflect.ReflectionFactory"); - Method getReflectionFactory = factoryClass.getDeclaredMethod("getReflectionFactory"); - reflectionFactory = getReflectionFactory.invoke(null); - writeObject = factoryClass.getDeclaredMethod("writeObjectForSerialization", Class.class); - readObject = factoryClass.getDeclaredMethod("readObjectForSerialization", Class.class); + MethodHandles.Lookup lookup = _JDKAccess._trustedLookup(factoryClass); + MethodHandle getReflectionFactory = + lookup.findStatic( + factoryClass, "getReflectionFactory", MethodType.methodType(factoryClass)); + reflectionFactory = getReflectionFactory.invoke(); + MethodType hookType = MethodType.methodType(MethodHandle.class, Class.class); + writeObject = lookup.findVirtual(factoryClass, "writeObjectForSerialization", hookType); + readObject = lookup.findVirtual(factoryClass, "readObjectForSerialization", hookType); readObjectNoData = - factoryClass.getDeclaredMethod("readObjectNoDataForSerialization", Class.class); + lookup.findVirtual(factoryClass, "readObjectNoDataForSerialization", hookType); try { defaultReadObject = - factoryClass.getDeclaredMethod("defaultReadObjectForSerialization", Class.class); - } catch (NoSuchMethodException e) { + lookup.findVirtual(factoryClass, "defaultReadObjectForSerialization", hookType); + } catch (NoSuchMethodException | IllegalAccessException e) { ExceptionUtils.ignore(e); } - writeReplace = - factoryClass.getDeclaredMethod("writeReplaceForSerialization", Class.class); - readResolve = factoryClass.getDeclaredMethod("readResolveForSerialization", Class.class); + writeReplace = lookup.findVirtual(factoryClass, "writeReplaceForSerialization", hookType); + readResolve = lookup.findVirtual(factoryClass, "readResolveForSerialization", hookType); } catch (Throwable e) { ExceptionUtils.ignore(e); } @@ -87,7 +91,7 @@ private static final class Methods { } } - private static MethodHandle getHandle(Class type, Method factoryMethod) { + private static MethodHandle getHandle(Class type, MethodHandle factoryMethod) { if (Methods.REFLECTION_FACTORY == null || factoryMethod == null) { return null; } @@ -99,7 +103,7 @@ private static MethodHandle getHandle(Class type, Method factoryMethod) { } } - private static Method getMethod(Class type, Method factoryMethod) { + private static Method getMethod(Class type, MethodHandle factoryMethod) { MethodHandle handle = getHandle(type, factoryMethod); return handle == null ? null : MethodHandles.reflectAs(Method.class, handle); } diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/ChildContainerSerializers.java b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/ChildContainerSerializers.java index 3f7670fd47..25f16464c0 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/ChildContainerSerializers.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/ChildContainerSerializers.java @@ -52,7 +52,7 @@ import org.apache.fory.exception.ForyException; import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.meta.TypeDef; -import org.apache.fory.reflect.ObjectCreator; +import org.apache.fory.reflect.ObjectInstantiator; import org.apache.fory.reflect.ReflectionUtils; import org.apache.fory.resolver.ClassResolver; import org.apache.fory.resolver.TypeResolver; @@ -74,10 +74,13 @@ */ @SuppressWarnings({"unchecked", "rawtypes"}) public class ChildContainerSerializers { - private static final ObjectCreator FIELD_ONLY_CREATOR = new FieldOnlyCreator(); + // Child-container slot serializers populate fields on the container instance created by the + // concrete container serializer. Parent slots do not own object construction. + private static final ObjectInstantiator FIELD_ONLY_INSTANTIATOR = + new FieldOnlyInstantiator(); - private static final class FieldOnlyCreator extends ObjectCreator { - private FieldOnlyCreator() { + private static final class FieldOnlyInstantiator extends ObjectInstantiator { + private FieldOnlyInstantiator() { super(Object.class); } @@ -644,7 +647,7 @@ private static Serializer[] buildSlotsSerializers( slotsSerializer = new CompatibleLayerSerializer(typeResolver, cls, layerTypeDef, layerMarkerClass); } else { - slotsSerializer = new ObjectSerializer<>(typeResolver, cls, false, slotCreator()); + slotsSerializer = new ObjectSerializer<>(typeResolver, cls, false, slotInstantiator()); } serializers.add(slotsSerializer); cls = (Class) cls.getSuperclass(); @@ -654,8 +657,8 @@ private static Serializer[] buildSlotsSerializers( return serializers.toArray(new Serializer[0]); } - private static ObjectCreator slotCreator() { - return (ObjectCreator) FIELD_ONLY_CREATOR; + private static ObjectInstantiator slotInstantiator() { + return (ObjectInstantiator) FIELD_ONLY_INSTANTIATOR; } private static void readAndSetFields( diff --git a/java/fory-core/src/main/java25/org/apache/fory/reflect/ConstructorBypassAllocator.java b/java/fory-core/src/main/java25/org/apache/fory/reflect/ConstructorBypassAllocator.java deleted file mode 100644 index 0a18869817..0000000000 --- a/java/fory-core/src/main/java25/org/apache/fory/reflect/ConstructorBypassAllocator.java +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.fory.reflect; - -import java.io.ObjectStreamClass; -import java.lang.invoke.MethodHandle; -import java.lang.invoke.MethodType; -import org.apache.fory.annotation.Internal; -import org.apache.fory.exception.ForyException; -import org.apache.fory.platform.internal._JDKAccess; - -/** JDK25 replacement for the JDK8-24 constructor-bypass allocator. */ -@Internal -final class ConstructorBypassAllocator { - private final Class type; - private final ObjectStreamClass objectStreamClass; - - ConstructorBypassAllocator(Class type) { - this.type = type; - objectStreamClass = ObjectStreamClass.lookupAny(type); - } - - T allocate() { - try { - return type.cast(ObjectStreamClassAccess.newInstance(objectStreamClass)); - } catch (Throwable e) { - throw handleAllocationException(type, e); - } - } - - private static RuntimeException handleAllocationException(Class type, Throwable cause) { - if (cause instanceof UnsupportedOperationException || cause instanceof InstantiationException) { - return unsupported(type, cause); - } - return new ForyException("Failed to create an instance for " + type, cause); - } - - private static ForyException unsupported(Class type, Throwable cause) { - return new ForyException( - "Cannot create a constructor-bypassing instance for " - + type - + " in JDK25+ zero-Unsafe mode. Provide an accessible no-arg constructor, " - + "use a record canonical constructor, or register a custom serializer.", - cause); - } - - private static final class ObjectStreamClassAccess { - private static final MethodHandle NEW_INSTANCE; - private static final Throwable INIT_ERROR; - - static { - MethodHandle newInstance = null; - Throwable error = null; - try { - newInstance = - _JDKAccess - ._trustedLookup(ObjectStreamClass.class) - .findVirtual( - ObjectStreamClass.class, "newInstance", MethodType.methodType(Object.class)); - } catch (ReflectiveOperationException | RuntimeException e) { - error = e; - } - NEW_INSTANCE = newInstance; - INIT_ERROR = error; - } - - private static Object newInstance(ObjectStreamClass objectStreamClass) throws Throwable { - MethodHandle handle = NEW_INSTANCE; - if (handle == null) { - throw missingLookup(); - } - return handle.invoke(objectStreamClass); - } - - private static ForyException missingLookup() { - return new ForyException( - "JDK25+ Serializable object creation requires java.base/java.lang.invoke to be open " - + "to org.apache.fory.core", - INIT_ERROR); - } - } -} diff --git a/java/fory-core/src/main/java25/org/apache/fory/reflect/UnsafeObjectInstantiator.java b/java/fory-core/src/main/java25/org/apache/fory/reflect/UnsafeObjectInstantiator.java new file mode 100644 index 0000000000..8f4c1ef374 --- /dev/null +++ b/java/fory-core/src/main/java25/org/apache/fory/reflect/UnsafeObjectInstantiator.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.reflect; + +import org.apache.fory.annotation.Internal; +import org.apache.fory.exception.ForyException; + +/** JDK25 replacement for the JDK8-24 Unsafe-backed instantiator. */ +@Internal +final class UnsafeObjectInstantiator extends ObjectInstantiator { + UnsafeObjectInstantiator(Class type) { + super(type); + } + + @Override + public T newInstance() { + throw unsupported(type); + } + + @Override + public T newInstanceWithArguments(Object... arguments) { + throw unsupported(type); + } + + private static ForyException unsupported(Class type) { + return new ForyException( + "Unsafe allocation is unsupported for " + + type + + " in JDK25+ zero-Unsafe mode. Provide an accessible no-arg constructor, " + + "use a record canonical constructor, or register a custom serializer."); + } +} diff --git a/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties b/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties index 559b47c218..653b6fb867 100644 --- a/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties +++ b/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties @@ -31,8 +31,7 @@ Args=--initialize-at-build-time=org.apache.fory.annotation.ForyField$Dynamic,\ org.apache.fory.memory.NativeByteOrder,\ org.apache.fory.platform.AndroidSupport,\ org.apache.fory.platform.internal._UnsafeUtils,\ - org.apache.fory.reflect.ConstructorBypassAllocator,\ - org.apache.fory.reflect.ObjectCreatorRegistry,\ + org.apache.fory.reflect.UnsafeObjectInstantiator,\ org.apache.fory.reflect.JvmTypeUseMetadata,\ org.apache.fory.reflect.TypeUseMetadata,\ org.apache.fory.resolver.StaticGeneratedSerializerRegistry,\ @@ -258,11 +257,10 @@ Args=--initialize-at-build-time=org.apache.fory.annotation.ForyField$Dynamic,\ org.apache.fory.reflect.TypeRef$ClassOwnership,\ org.apache.fory.reflect.TypeRef$TypeVariableKey,\ org.apache.fory.reflect.TypeRef,\ - org.apache.fory.reflect.ObjectCreators,\ - org.apache.fory.reflect.ObjectCreators$ConstructorBypassObjectCreator,\ - org.apache.fory.reflect.ObjectCreators$DeclaredNoArgCtrObjectCreator,\ - org.apache.fory.reflect.ObjectCreators$ParentNoArgCtrObjectCreator,\ - org.apache.fory.reflect.ObjectCreators$RecordObjectCreator,\ + org.apache.fory.reflect.ObjectInstantiators,\ + org.apache.fory.reflect.ObjectInstantiators$DeclaredNoArgCtrInstantiator,\ + org.apache.fory.reflect.ObjectInstantiators$ParentNoArgCtrInstantiator,\ + org.apache.fory.reflect.ObjectInstantiators$RecordObjectInstantiator,\ org.apache.fory.resolver.TypeChecker,\ org.apache.fory.resolver.TypeInfo,\ org.apache.fory.resolver.TypeInfoHolder,\ @@ -339,11 +337,11 @@ Args=--initialize-at-build-time=org.apache.fory.annotation.ForyField$Dynamic,\ org.apache.fory.serializer.BufferSerializers$ByteBufferSerializer,\ org.apache.fory.serializer.CompatibleSerializer,\ org.apache.fory.serializer.CompatibleLayerSerializer,\ - org.apache.fory.serializer.CompatibleLayerSerializerBase$FieldOnlyCreator,\ + org.apache.fory.serializer.CompatibleLayerSerializerBase$FieldOnlyInstantiator,\ org.apache.fory.serializer.EnumSerializer,\ org.apache.fory.serializer.ExceptionSerializers,\ org.apache.fory.serializer.ExceptionSerializers$ExceptionSerializer,\ - org.apache.fory.serializer.ExceptionSerializers$FieldOnlyCreator,\ + org.apache.fory.serializer.ExceptionSerializers$FieldOnlyInstantiator,\ org.apache.fory.serializer.ExceptionSerializers$StackTraceElementSerializer,\ org.apache.fory.serializer.collection.SubListSerializers,\ org.apache.fory.serializer.collection.SubListSerializers$SubListSerializer,\ @@ -501,7 +499,7 @@ Args=--initialize-at-build-time=org.apache.fory.annotation.ForyField$Dynamic,\ org.apache.fory.serializer.ObjectStreamSerializer$StreamTypeInfo,\ org.apache.fory.serializer.ObjectStreamSerializer$SlotsInfo,\ org.apache.fory.serializer.collection.ChildContainerSerializers$ChildCollectionSerializer,\ - org.apache.fory.serializer.collection.ChildContainerSerializers$FieldOnlyCreator,\ + org.apache.fory.serializer.collection.ChildContainerSerializers$FieldOnlyInstantiator,\ org.apache.fory.serializer.collection.ChildContainerSerializers$ChildMapSerializer,\ org.apache.fory.builder.LayerMarkerClassGenerator,\ org.apache.fory.builder.LayerMarkerClassGenerator$1,\ diff --git a/java/fory-core/src/test/java/org/apache/fory/TestUtils.java b/java/fory-core/src/test/java/org/apache/fory/TestUtils.java index 321ac38f71..48cef098dc 100644 --- a/java/fory-core/src/test/java/org/apache/fory/TestUtils.java +++ b/java/fory-core/src/test/java/org/apache/fory/TestUtils.java @@ -46,7 +46,7 @@ import org.apache.fory.platform.JdkVersion; import org.apache.fory.platform.internal._JDKAccess; import org.apache.fory.reflect.FieldAccessor; -import org.apache.fory.reflect.ObjectCreators; +import org.apache.fory.reflect.ObjectInstantiators; import org.apache.fory.reflect.ReflectionUtils; import org.apache.fory.type.Descriptor; import org.testng.SkipException; @@ -180,7 +180,7 @@ public static void jdkSerialize(ByteArrayOutputStream bas, Object data) { public static T unsafeCopy(T obj) { @SuppressWarnings("unchecked") - T newInstance = (T) ObjectCreators.getObjectCreator(obj.getClass()).newInstance(); + T newInstance = (T) ObjectInstantiators.getObjectInstantiator(obj.getClass()).newInstance(); for (Field field : ReflectionUtils.getFields(obj.getClass(), true)) { if (!Modifier.isStatic(field.getModifiers())) { // Don't cache accessors by `obj.getClass()` using WeakHashMap, the `field` will reference diff --git a/java/fory-core/src/test/java/org/apache/fory/builder/Jdk25MultiReleaseJarVerifier.java b/java/fory-core/src/test/java/org/apache/fory/builder/Jdk25MultiReleaseJarVerifier.java index 33b29a7fbb..bf0f9faf7c 100644 --- a/java/fory-core/src/test/java/org/apache/fory/builder/Jdk25MultiReleaseJarVerifier.java +++ b/java/fory-core/src/test/java/org/apache/fory/builder/Jdk25MultiReleaseJarVerifier.java @@ -40,9 +40,7 @@ public final class Jdk25MultiReleaseJarVerifier { private static final String VERSION_25_PREFIX = "META-INF/versions/25/"; private static final String FORY_CLASS_PREFIX = "org/apache/fory/"; private static final String SHADED_CLASS_PREFIX = "org/apache/fory/shaded/"; - private static final String[] FORBIDDEN_CONSTANTS = { - "sun/misc/Unsafe", "sun.misc.Unsafe", "jdk/internal/reflect", "jdk.internal.reflect" - }; + private static final String[] FORBIDDEN_CONSTANTS = {"sun/misc/Unsafe", "sun.misc.Unsafe"}; private static final String[] REQUIRED_VERSION_25_CLASSES = { "module-info.class", "org/apache/fory/memory/LittleEndian.class", @@ -51,7 +49,7 @@ public final class Jdk25MultiReleaseJarVerifier { "org/apache/fory/platform/internal/_UnsafeUtils.class", "org/apache/fory/builder/UnsafeCodegenSupport.class", "org/apache/fory/reflect/InstanceFieldAccessors.class", - "org/apache/fory/reflect/ConstructorBypassAllocator.class", + "org/apache/fory/reflect/UnsafeObjectInstantiator.class", "org/apache/fory/serializer/PlatformStringUtils.class" }; diff --git a/java/fory-core/src/test/java/org/apache/fory/reflect/ObjectCreatorsTest.java b/java/fory-core/src/test/java/org/apache/fory/reflect/ObjectInstantiatorsTest.java similarity index 74% rename from java/fory-core/src/test/java/org/apache/fory/reflect/ObjectCreatorsTest.java rename to java/fory-core/src/test/java/org/apache/fory/reflect/ObjectInstantiatorsTest.java index bda955b4c4..315d8654f9 100644 --- a/java/fory-core/src/test/java/org/apache/fory/reflect/ObjectCreatorsTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/reflect/ObjectInstantiatorsTest.java @@ -28,12 +28,12 @@ import org.apache.fory.exception.ForyException; import org.apache.fory.platform.AndroidSupport; import org.apache.fory.platform.JdkVersion; -import org.apache.fory.reflect.ObjectCreators.ParentNoArgCtrObjectCreator; +import org.apache.fory.reflect.ObjectInstantiators.ParentNoArgCtrInstantiator; import org.testng.Assert; import org.testng.annotations.Test; @SuppressWarnings("rawtypes") -public class ObjectCreatorsTest { +public class ObjectInstantiatorsTest { static class NoCtrTestClass { int f1; @@ -44,22 +44,22 @@ public NoCtrTestClass(int f1) { } @Test - public void testObjectCreator() { + public void testObjectInstantiator() { if (JdkVersion.MAJOR_VERSION >= 25) { return; } - ParentNoArgCtrObjectCreator creator = - new ParentNoArgCtrObjectCreator<>(ArrayBlockingQueue.class); - Assert.assertEquals(creator.newInstance().getClass(), ArrayBlockingQueue.class); + ParentNoArgCtrInstantiator instantiator = + new ParentNoArgCtrInstantiator<>(ArrayBlockingQueue.class); + Assert.assertEquals(instantiator.newInstance().getClass(), ArrayBlockingQueue.class); Assert.assertEquals( - new ParentNoArgCtrObjectCreator<>(NoCtrTestClass.class).newInstance().getClass(), + new ParentNoArgCtrInstantiator<>(NoCtrTestClass.class).newInstance().getClass(), NoCtrTestClass.class); } @Test - public void testAndroidObjectCreators() throws Exception { + public void testAndroidObjectInstantiators() throws Exception { Process process = - new ProcessBuilder(TestUtils.javaCommand(AndroidObjectCreatorProbe.class)) + new ProcessBuilder(TestUtils.javaCommand(AndroidObjectInstantiatorProbe.class)) .redirectErrorStream(true) .start(); String output = readFully(process.getInputStream()); @@ -76,22 +76,22 @@ private static String readFully(InputStream inputStream) throws IOException { return new String(outputStream.toByteArray(), StandardCharsets.UTF_8); } - public static final class AndroidObjectCreatorProbe { + public static final class AndroidObjectInstantiatorProbe { public static void main(String[] args) { System.setProperty("java.vm.name", "Dalvik"); System.setProperty("java.runtime.name", "Android Runtime"); check(AndroidSupport.IS_ANDROID, "AndroidSupport should detect Dalvik runtime"); - ObjectCreator creator = - ObjectCreators.getObjectCreator(AndroidPrivateNoArg.class); - AndroidPrivateNoArg instance = creator.newInstance(); + ObjectInstantiator instantiator = + ObjectInstantiators.getObjectInstantiator(AndroidPrivateNoArg.class); + AndroidPrivateNoArg instance = instantiator.newInstance(); check(instance.value == 7, "Android reflective constructor should initialize fields"); - ObjectCreator unsupported = - ObjectCreators.getObjectCreator(AndroidNoNoArg.class); + ObjectInstantiator unsupported = + ObjectInstantiators.getObjectInstantiator(AndroidNoNoArg.class); try { unsupported.newInstance(); - throw new AssertionError("Android creator without no-arg constructor should fail"); + throw new AssertionError("Android instantiator without no-arg constructor should fail"); } catch (ForyException expected) { check( expected.getMessage().contains("without an accessible no-arg constructor"), diff --git a/java/fory-core/src/test/java/org/apache/fory/reflect/ReflectionUtilsTest.java b/java/fory-core/src/test/java/org/apache/fory/reflect/ReflectionUtilsTest.java index b00d3e54b7..6f8d862ec4 100644 --- a/java/fory-core/src/test/java/org/apache/fory/reflect/ReflectionUtilsTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/reflect/ReflectionUtilsTest.java @@ -118,7 +118,7 @@ public void testGetNoArgConstructor() throws Throwable { Assert.assertNull(ctr); // ReflectionFactory serialization constructors are invoked directly by - // ParentNoArgCtrObjectCreator. + // ParentNoArgCtrInstantiator. // MethodHandle handle = lookup.unreflectConstructor(ctr); // System.out.println(ctr); // System.out.println(handle); diff --git a/java/fory-core/src/test/java/org/apache/fory/serializer/ObjectSerializerTest.java b/java/fory-core/src/test/java/org/apache/fory/serializer/ObjectSerializerTest.java index 265c246502..b505cda775 100644 --- a/java/fory-core/src/test/java/org/apache/fory/serializer/ObjectSerializerTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/serializer/ObjectSerializerTest.java @@ -21,7 +21,6 @@ import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNotSame; -import static org.testng.Assert.assertSame; import java.io.ByteArrayOutputStream; import java.io.IOException; diff --git a/java/fory-core/src/test/java/org/apache/fory/serializer/collection/CollectionSerializersTest.java b/java/fory-core/src/test/java/org/apache/fory/serializer/collection/CollectionSerializersTest.java index b2bf9029c7..23129899b9 100644 --- a/java/fory-core/src/test/java/org/apache/fory/serializer/collection/CollectionSerializersTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/serializer/collection/CollectionSerializersTest.java @@ -260,14 +260,14 @@ public void testSortedSet(boolean referenceTrackingConfig) { // Test serialize Comparator TreeSet set = new TreeSet<>( - (s1, s2) -> { - int delta = s1.length() - s2.length(); - if (delta == 0) { - return s1.compareTo(s2); - } else { - return delta; - } - }); + (s1, s2) -> { + int delta = s1.length() - s2.length(); + if (delta == 0) { + return s1.compareTo(s2); + } else { + return delta; + } + }); set.add("str11"); set.add("str2"); assertEquals(set, serDe(fory, set)); diff --git a/java/fory-core/src/test/java/org/apache/fory/serializer/collection/MapSerializersTest.java b/java/fory-core/src/test/java/org/apache/fory/serializer/collection/MapSerializersTest.java index 34db014535..bfd83c6264 100644 --- a/java/fory-core/src/test/java/org/apache/fory/serializer/collection/MapSerializersTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/serializer/collection/MapSerializersTest.java @@ -176,14 +176,14 @@ public void basicTestCaseWithMultiConfig( // testTreeMap TreeMap map = new TreeMap<>( - (s1, s2) -> { - int delta = s1.length() - s2.length(); - if (delta == 0) { - return s1.compareTo(s2); - } else { - return delta; - } - }); + (s1, s2) -> { + int delta = s1.length() - s2.length(); + if (delta == 0) { + return s1.compareTo(s2); + } else { + return delta; + } + }); map.put("str1", "1"); map.put("str2", "1"); assertEquals(map, serDe(fory, map)); @@ -379,14 +379,14 @@ public void testTreeMap() { .build(); TreeMap map = new TreeMap<>( - (s1, s2) -> { - int delta = s1.length() - s2.length(); - if (delta == 0) { - return s1.compareTo(s2); - } else { - return delta; - } - }); + (s1, s2) -> { + int delta = s1.length() - s2.length(); + if (delta == 0) { + return s1.compareTo(s2); + } else { + return delta; + } + }); map.put("str1", "1"); map.put("str2", "1"); assertEquals(map, serDe(fory, map)); @@ -398,14 +398,14 @@ public void testTreeMap() { public void testTreeMap(Fory fory) { TreeMap map = new TreeMap<>( - (s1, s2) -> { - int delta = s1.length() - s2.length(); - if (delta == 0) { - return s1.compareTo(s2); - } else { - return delta; - } - }); + (s1, s2) -> { + int delta = s1.length() - s2.length(); + if (delta == 0) { + return s1.compareTo(s2); + } else { + return delta; + } + }); map.put("str1", "1"); map.put("str2", "1"); copyCheck(fory, map); diff --git a/kotlin/fory-kotlin-ksp/src/main/kotlin/org/apache/fory/kotlin/ksp/KotlinSerializerSourceWriter.kt b/kotlin/fory-kotlin-ksp/src/main/kotlin/org/apache/fory/kotlin/ksp/KotlinSerializerSourceWriter.kt index e84d7b7b09..ae784daf34 100644 --- a/kotlin/fory-kotlin-ksp/src/main/kotlin/org/apache/fory/kotlin/ksp/KotlinSerializerSourceWriter.kt +++ b/kotlin/fory-kotlin-ksp/src/main/kotlin/org/apache/fory/kotlin/ksp/KotlinSerializerSourceWriter.kt @@ -276,6 +276,9 @@ internal class KotlinSerializerSourceWriter(private val struct: KotlinSourceStru builder.append(" }\n") builder.append(" }\n\n") } + if (struct.construction != KotlinStructConstruction.CONSTRUCTOR) { + return + } builder .append(" private fun readCompatibleConstructor(readContext: ReadContext): ") @@ -1152,10 +1155,10 @@ internal class KotlinSerializerSourceWriter(private val struct: KotlinSourceStru private fun constructorFieldValueExpression(field: KotlinSourceField): String { val source = "fieldValues[${field.id}]" - if (field.type.valueTypeName == "Any?") { + if (field.propertyTypeName == "Any?") { return source } - return "($source as ${field.type.valueTypeName})" + return "($source as ${field.propertyTypeName})" } private fun constructorValueExpression(field: KotlinSourceField): String { diff --git a/kotlin/fory-kotlin-ksp/src/test/kotlin/org/apache/fory/kotlin/ksp/ProcessorValidationTest.kt b/kotlin/fory-kotlin-ksp/src/test/kotlin/org/apache/fory/kotlin/ksp/ProcessorValidationTest.kt index 3498fdf932..48a5ea22ac 100644 --- a/kotlin/fory-kotlin-ksp/src/test/kotlin/org/apache/fory/kotlin/ksp/ProcessorValidationTest.kt +++ b/kotlin/fory-kotlin-ksp/src/test/kotlin/org/apache/fory/kotlin/ksp/ProcessorValidationTest.kt @@ -462,7 +462,7 @@ class ProcessorValidationTest { assertTrue( source.contains( - "return User(counts = (fieldValues[0] as kotlin.collections.Map), names = (fieldValues[1] as kotlin.collections.List>), arrays = (fieldValues[2] as kotlin.collections.List), nestedCounts = (fieldValues[3] as java.util.TreeMap>))" + "return User(counts = (fieldValues[0] as java.util.TreeMap), names = (fieldValues[1] as java.util.List>), arrays = (fieldValues[2] as java.util.List), nestedCounts = (fieldValues[3] as java.util.TreeMap>))" ) ) assertTrue( @@ -867,6 +867,8 @@ class ProcessorValidationTest { assertTrue(source.contains("presentMask0 = presentMask0 or (1L shl 0)")) assertTrue(source.contains("Required Kotlin field example.Node.id is missing")) assertTrue(source.contains("copyContext.reference(value, copy)")) + assertFalse(source.contains("readCompatibleConstructor(")) + assertFalse(source.contains("newConstructorObject(")) assertTrue(!source.contains("return Node(parent =")) } diff --git a/kotlin/fory-kotlin/src/main/kotlin/org/apache/fory/serializer/kotlin/KotlinDefaultValueSupport.kt b/kotlin/fory-kotlin/src/main/kotlin/org/apache/fory/serializer/kotlin/KotlinDefaultValueSupport.kt index 3cd17bdff7..dff2ebce21 100644 --- a/kotlin/fory-kotlin/src/main/kotlin/org/apache/fory/serializer/kotlin/KotlinDefaultValueSupport.kt +++ b/kotlin/fory-kotlin/src/main/kotlin/org/apache/fory/serializer/kotlin/KotlinDefaultValueSupport.kt @@ -31,7 +31,7 @@ import org.apache.fory.collection.ClassValueCache import org.apache.fory.logging.Logger import org.apache.fory.logging.LoggerFactory import org.apache.fory.platform.AndroidSupport -import org.apache.fory.reflect.ObjectCreators +import org.apache.fory.reflect.ObjectInstantiators import org.apache.fory.util.DefaultValueUtils /** @@ -181,7 +181,7 @@ internal class KotlinDefaultValueSupport : DefaultValueUtils.DefaultValueSupport } private fun newDefaultInstance(clazz: Class<*>): Any? { - return ObjectCreators.getObjectCreator(clazz).newInstance() + return ObjectInstantiators.getObjectInstantiator(clazz).newInstance() } private fun newPublicDefaultInstance( From debe2062c612702094650b3ec0c2f97472b6d18c Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Tue, 2 Jun 2026 22:39:05 +0800 Subject: [PATCH 61/69] fix(java): support nonserializable reflection instantiation --- .agents/languages/java.md | 8 ++- .../model/NonSerializableNoNoArgBean.java | 55 +++++++++++++++++++ .../JpmsFieldAccessorTest.java | 17 ++++++ .../fory/reflect/ObjectInstantiators.java | 6 ++ .../fory/reflect/ObjectInstantiatorsTest.java | 34 ++++++++++++ 5 files changed, 117 insertions(+), 3 deletions(-) create mode 100644 integration_tests/jpms_tests/src/main/java/org/apache/fory/integration_tests/model/NonSerializableNoNoArgBean.java diff --git a/.agents/languages/java.md b/.agents/languages/java.md index 27a08f8985..182829979f 100644 --- a/.agents/languages/java.md +++ b/.agents/languages/java.md @@ -69,9 +69,11 @@ Load this file when changing anything under `java/` or when Java drives a cross- through `ParentNoArgCtrInstantiator`, whose JDK25+ path owns trusted-lookup access to `jdk.internal.reflect.ReflectionFactory` in `java.base`. This must not require `--add-opens=java.base/jdk.internal.reflect=...`; the only JDK25+ platform open remains - `java.base/java.lang.invoke=org.apache.fory.core`. Classes unsupported by the serialization - constructor model require an accessible no-arg constructor, a record canonical constructor path, - or a custom serializer. + `java.base/java.lang.invoke=org.apache.fory.core`. ReflectionFactory serialization constructors + also support non-Serializable ordinary classes; the normal object-instantiation path must not + reject them with ObjectStream-only parent-constructor checks. Classes unsupported by + ReflectionFactory itself require an accessible no-arg constructor, a record canonical constructor + path, or a custom serializer. - `UnsafeObjectInstantiator` is the JDK8-24 Unsafe owner only. It must be a top-level instantiator with a Java25 multi-release stub that contains no Unsafe, ObjectStream, ReflectionFactory, or constructor-bypass implementation. diff --git a/integration_tests/jpms_tests/src/main/java/org/apache/fory/integration_tests/model/NonSerializableNoNoArgBean.java b/integration_tests/jpms_tests/src/main/java/org/apache/fory/integration_tests/model/NonSerializableNoNoArgBean.java new file mode 100644 index 0000000000..2f44ada084 --- /dev/null +++ b/integration_tests/jpms_tests/src/main/java/org/apache/fory/integration_tests/model/NonSerializableNoNoArgBean.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.integration_tests.model; + +class NonSerializableNoNoArgParent { + static int constructorCalls; + final int parentValue; + + NonSerializableNoNoArgParent(int parentValue) { + constructorCalls++; + this.parentValue = parentValue; + } +} + +public final class NonSerializableNoNoArgBean extends NonSerializableNoNoArgParent { + private final int value; + + public NonSerializableNoNoArgBean(int parentValue, int value) { + super(parentValue); + this.value = value; + } + + public int parentValue() { + return parentValue; + } + + public int value() { + return value; + } + + public static void resetParentConstructorCalls() { + NonSerializableNoNoArgParent.constructorCalls = 0; + } + + public static int parentConstructorCalls() { + return NonSerializableNoNoArgParent.constructorCalls; + } +} diff --git a/integration_tests/jpms_tests/src/test/java/org/apache/fory/integration_tests/JpmsFieldAccessorTest.java b/integration_tests/jpms_tests/src/test/java/org/apache/fory/integration_tests/JpmsFieldAccessorTest.java index 1381233d8d..c9414385ff 100644 --- a/integration_tests/jpms_tests/src/test/java/org/apache/fory/integration_tests/JpmsFieldAccessorTest.java +++ b/integration_tests/jpms_tests/src/test/java/org/apache/fory/integration_tests/JpmsFieldAccessorTest.java @@ -21,6 +21,7 @@ import java.lang.reflect.Field; import org.apache.fory.Fory; +import org.apache.fory.integration_tests.model.NonSerializableNoNoArgBean; import org.apache.fory.integration_tests.model.PrivateFieldBean; import org.apache.fory.integration_tests.publicserializer.PublicSerializerValue; import org.apache.fory.integration_tests.publicserializer.PublicSerializerValueSerializer; @@ -54,6 +55,22 @@ public void testPrivateFinalFieldSerialization() { Assert.assertEquals(result.value(), 13); } + @Test + public void testNonSerializableNoNoArgSerialization() { + if (JDK_MAJOR_VERSION < 25) { + return; + } + Fory fory = + Fory.builder().withXlang(false).withCodegen(false).requireClassRegistration(false).build(); + byte[] bytes = fory.serialize(new NonSerializableNoNoArgBean(5, 13)); + NonSerializableNoNoArgBean.resetParentConstructorCalls(); + NonSerializableNoNoArgBean result = + (NonSerializableNoNoArgBean) fory.deserialize(bytes); + Assert.assertEquals(result.parentValue(), 5); + Assert.assertEquals(result.value(), 13); + Assert.assertEquals(NonSerializableNoNoArgBean.parentConstructorCalls(), 0); + } + @Test public void testCodegenFinalFieldAccess() throws Exception { if (JDK_MAJOR_VERSION < 25) { diff --git a/java/fory-core/src/main/java/org/apache/fory/reflect/ObjectInstantiators.java b/java/fory-core/src/main/java/org/apache/fory/reflect/ObjectInstantiators.java index aae9484e88..672f4588ba 100644 --- a/java/fory-core/src/main/java/org/apache/fory/reflect/ObjectInstantiators.java +++ b/java/fory-core/src/main/java/org/apache/fory/reflect/ObjectInstantiators.java @@ -289,6 +289,12 @@ private static Constructor createSerializationConstructor(Class type) private static Constructor findSerializationConstructor(Class type) throws NoSuchMethodException { + if (!Serializable.class.isAssignableFrom(type)) { + // ReflectionFactory can synthesize serialization constructors for ordinary classes too. + // Use Object as the template so normal Fory object creation keeps empty-instance + // semantics and never depends on the target class hierarchy exposing a no-arg constructor. + return Object.class.getDeclaredConstructor(); + } Class current = type.getSuperclass(); // Java ObjectStream reconstruction skips every Serializable class constructor and invokes // only the first non-Serializable superclass no-arg constructor. diff --git a/java/fory-core/src/test/java/org/apache/fory/reflect/ObjectInstantiatorsTest.java b/java/fory-core/src/test/java/org/apache/fory/reflect/ObjectInstantiatorsTest.java index 315d8654f9..dd4f12f5e4 100644 --- a/java/fory-core/src/test/java/org/apache/fory/reflect/ObjectInstantiatorsTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/reflect/ObjectInstantiatorsTest.java @@ -43,6 +43,25 @@ public NoCtrTestClass(int f1) { } } + static class NonSerializableParentWithoutNoArg { + static int constructorCalls; + int parentValue; + + NonSerializableParentWithoutNoArg(int parentValue) { + constructorCalls++; + this.parentValue = parentValue; + } + } + + static class NonSerializableChildWithoutNoArg extends NonSerializableParentWithoutNoArg { + int childValue; + + NonSerializableChildWithoutNoArg(int parentValue, int childValue) { + super(parentValue); + this.childValue = childValue; + } + } + @Test public void testObjectInstantiator() { if (JdkVersion.MAJOR_VERSION >= 25) { @@ -56,6 +75,21 @@ public void testObjectInstantiator() { NoCtrTestClass.class); } + @Test + public void testNonSerializableInstantiator() { + if (JdkVersion.MAJOR_VERSION >= 25) { + return; + } + NonSerializableParentWithoutNoArg.constructorCalls = 0; + ParentNoArgCtrInstantiator instantiator = + new ParentNoArgCtrInstantiator<>(NonSerializableChildWithoutNoArg.class); + NonSerializableChildWithoutNoArg instance = instantiator.newInstance(); + Assert.assertEquals(instance.getClass(), NonSerializableChildWithoutNoArg.class); + Assert.assertEquals(NonSerializableParentWithoutNoArg.constructorCalls, 0); + Assert.assertEquals(instance.parentValue, 0); + Assert.assertEquals(instance.childValue, 0); + } + @Test public void testAndroidObjectInstantiators() throws Exception { Process process = From f5105ce2197ee9580d19b6d2d2f1cf52e9949b33 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Tue, 2 Jun 2026 23:04:13 +0800 Subject: [PATCH 62/69] refactor(java): simplify fory-core mrjar build --- java/fory-core/pom.xml | 337 ++++++++++++++--------------------------- 1 file changed, 111 insertions(+), 226 deletions(-) diff --git a/java/fory-core/pom.xml b/java/fory-core/pom.xml index 521af568a8..2f3a26283a 100644 --- a/java/fory-core/pom.xml +++ b/java/fory-core/pom.xml @@ -39,6 +39,7 @@ 8 8 ${basedir}/.. + ${project.build.directory}/multi-release-classes @@ -147,39 +148,39 @@ org.apache.maven.plugins - maven-antrun-plugin - 3.1.0 + maven-compiler-plugin compile-java9-module-info process-classes - run + compile - - - - - - - - - - - - - + 9 + none + + ${project.basedir}/src/main/java9 + + + module-info.java + + ${fory.mr.classes}/9 + + --patch-module + org.apache.fory.core=${project.build.outputDirectory} + + + + + org.apache.maven.plugins + maven-antrun-plugin + 3.1.0 + - inject-java9-module-info + inject-multi-release-classes package run @@ -187,40 +188,15 @@ - + - - org.apache.maven.plugins - maven-resources-plugin - 3.3.1 - - - copy-java9-module-info - prepare-package - - copy-resources - - - ${project.build.outputDirectory}/META-INF/versions/9 - - - ${project.build.directory}/jpms-classes/java9 - - module-info.class - - - - - - - @@ -233,93 +209,31 @@ org.apache.maven.plugins - maven-antrun-plugin - 3.1.0 + maven-compiler-plugin compile-java16-sources process-classes - run + compile - - - - - - - - - - - - - - - - - - - - - - clean-java16-package-classes - prepare-package - - run - - - - - - - - - inject-java16-classes - package - - run - - - - - - - - - - - - - org.apache.maven.plugins - maven-resources-plugin - 3.3.1 - - - copy-java16-classes - prepare-package - - copy-resources - - - ${project.build.outputDirectory}/META-INF/versions/16 - - - ${project.build.directory}/jpms-classes/java16 - - **/*.class - - - + 16 + 16 + none + + ${project.basedir}/src/main/java16 + + + **/*.java + + ${fory.mr.classes}/16 + + --add-modules + jdk.incubator.vector + --patch-module + org.apache.fory.core=${project.build.outputDirectory} + @@ -349,72 +263,62 @@ org.apache.maven.plugins - maven-antrun-plugin - 3.1.0 + maven-compiler-plugin - compile-jdk25-multi-release + compile-jdk25-classes process-classes - run + compile - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + 22 + none + + ${project.basedir}/src/main/java25 + + + org/**/*.java + + ${fory.mr.classes}/25 + + -sourcepath + + + + + + compile-jdk25-module-info + process-classes + + compile + + + 22 + 22 + none + + ${project.basedir}/src/main/java25 + + + module-info.java + + ${fory.mr.classes}/25 + + --add-modules + jdk.incubator.vector + --patch-module + org.apache.fory.core=${project.build.outputDirectory}${path.separator}${fory.mr.classes}/25 + + + + + org.apache.maven.plugins + maven-antrun-plugin + 3.1.0 + verify-jdk25-multi-release-jar verify @@ -436,7 +340,7 @@ - patch-jdk25-source-jar + patch-multi-release-source-jar package run @@ -449,8 +353,14 @@ + message="Multi-release source patching requires ${jdk25.sources.jar}; run with source-jar generation enabled."/> + + + + + + @@ -462,51 +372,26 @@ - - - - - + + - + file="${jdk25.sources.check.dir}/META-INF/versions/9/module-info.java" + property="java9.moduleinfo.source.present"/> - - + file="${jdk25.sources.check.dir}/META-INF/versions/16/module-info.java" + property="java16.moduleinfo.source.present"/> - - - + unless="java9.moduleinfo.source.present" + message="Java 9 module-info.java source is missing from the source jar."/> + unless="java16.moduleinfo.source.present" + message="Java 16 module-info.java source is missing from the source jar."/> From e83b31fa8e13a57fddcac1cc09f769d533066669 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Wed, 3 Jun 2026 00:28:29 +0800 Subject: [PATCH 63/69] clean stale code/docs/tests --- .agents/languages/java.md | 14 +++- README.md | 7 ++ docs/guide/java/index.md | 27 +++++++ docs/guide/java/native-serialization.md | 26 +++---- docs/guide/java/troubleshooting.md | 24 +----- docs/guide/kotlin/configuration.md | 12 --- docs/guide/kotlin/index.md | 9 +++ .../kotlin/static-generated-serializers.md | 12 +-- docs/guide/scala/index.md | 9 +++ .../org/apache/fory/context/ReadContext.java | 18 ++--- .../org/apache/fory/memory/MemoryUtils.java | 26 ++----- .../serializer/AbstractObjectSerializer.java | 16 +++- .../CompatibleLayerSerializerBase.java | 30 +------- .../fory/serializer/ExceptionSerializers.java | 29 +------- .../StaticGeneratedStructSerializer.java | 31 +------- .../collection/ChildContainerSerializers.java | 30 +------- .../collection/CollectionSerializers.java | 26 ++++--- .../fory-core/native-image.properties | 3 - .../collection/CollectionSerializersTest.java | 17 +++-- .../ksp/KotlinSerializerSourceWriter.kt | 31 ++++++-- .../kotlin/ksp/ProcessorValidationTest.kt | 73 +++++++++++++++---- 21 files changed, 231 insertions(+), 239 deletions(-) diff --git a/.agents/languages/java.md b/.agents/languages/java.md index 182829979f..8910192b8c 100644 --- a/.agents/languages/java.md +++ b/.agents/languages/java.md @@ -59,9 +59,19 @@ Load this file when changing anything under `java/` or when Java drives a cross- - If changes touch GraalVM bootstrap, serializer retention, native-image metadata, or `ObjectStreamSerializer` GraalVM behavior, verify the native-image build and run the produced binary; a plain Java compile is insufficient. - Put latest-JDK or virtual-thread tests in the latest-JDK test modules with the matching compiler/profile floor, and centralize runtime-version probing in existing compatibility utilities. - For JDK25+ zero-Unsafe work, preserve serializer-family selection by type and configuration. Do not switch a type from `ObjectStreamSerializer` or another Fory serializer family to `JavaSerializer`, a JDK stream fallback, or any broad `java.* Serializable` fallback by JDK version or no-arg-constructor shape. -- JDK25+ zero-Unsafe runtime support is a JPMS named-module design. Fory core must run with `--add-opens=java.base/java.lang.invoke=org.apache.fory.core` so it can obtain the trusted lookup; missing this open is an invalid access configuration, not a reason to open per-package JDK internals or switch serializer/object-creation families. Do not use `ALL-UNNAMED` as the zero-Unsafe verification target. +- JDK25+ zero-Unsafe runtime support is a JPMS named-module design. Fory core must run with `--add-opens=java.base/java.lang.invoke=org.apache.fory.core` so it can create a true trusted lookup through the private `MethodHandles.Lookup` constructor; missing this open is an invalid access configuration, not a reason to open per-package JDK internals or switch serializer/object-creation families. Do not use `ALL-UNNAMED` as the zero-Unsafe verification target. - Do not probe JDK25+ trusted-lookup availability and turn `_JDKAccess` field-access booleans false when the required `java.base/java.lang.invoke` open is missing. Keep those access flags true on JDK25+ and let the owning trusted-lookup path raise the configuration error. -- Keep JDK25+ unsafe-removal implementation invariants in agent/design docs and tests, not user guides. User guides should document user actions such as `--sun-misc-unsafe-memory-access=deny`, `java.base/java.lang.invoke` opens, and package opens for user named modules; do not expose internal serializer names, owner-model rationale, or avoided fallback strategies there. +- Keep JDK25+ unsafe-removal implementation invariants in agent/design docs and tests, not user guides. User guides should document user actions such as `--sun-misc-unsafe-memory-access=deny` and `java.base/java.lang.invoke` opens; do not expose internal serializer names, owner-model rationale, or avoided fallback strategies there. +- JDK25+ user docs must not require application module package opens for Fory private-field access. The only required JPMS open is `java.base/java.lang.invoke` to `org.apache.fory.core`; application module package opens are not part of this design. +- JDK25+ final-field user docs must not tell ordinary classes to implement + `java.io.Serializable`. Fory supports ordinary non-Serializable classes; mention + `Serializable` only for JDK serialization hook examples or `java.*` serializability checks. +- JDK25+ final-field user docs must not include fallback advice such as switching unsupported + classes to records, no-arg constructors, or custom serializers. Keep user docs focused on the + supported runtime setup and normal class model. +- Do not create a separate JDK25+ support user-guide page for the Java runtime setup unless the + user explicitly asks for one. Keep the `java.base/java.lang.invoke` open in install-facing docs + such as the Java/Kotlin/Scala install sections and README. - JDK25+ zero-Unsafe final-field writes must use a true target-class trusted lookup from the original `IMPL_LOOKUP`, not `IMPL_LOOKUP.in(type)`. JDK26+ normal Fory final-field restoration must pass with `--illegal-final-field-mutation=deny` and must not require `--enable-final-field-mutation`. - For JDK25+ object creation, do not use `sun.reflect.ReflectionFactory`, `jdk.unsupported`, `ObjectStreamClass.newInstance`, or an Unsafe-backed object instantiator. diff --git a/README.md b/README.md index a2f8502541..a05f50807b 100644 --- a/README.md +++ b/README.md @@ -142,6 +142,13 @@ Gradle: implementation "org.apache.fory:fory-core:1.1.0" ``` +On JDK25+, put Fory on the module path and open `java.lang.invoke` to the Fory +core module: + +```bash +--add-opens=java.base/java.lang.invoke=org.apache.fory.core +``` + **Scala** sbt: diff --git a/docs/guide/java/index.md b/docs/guide/java/index.md index c52237d803..5e07e611c0 100644 --- a/docs/guide/java/index.md +++ b/docs/guide/java/index.md @@ -50,6 +50,33 @@ Apache Fory™ provides blazingly fast Java object serialization with JIT compil - **Deep Copy**: Efficient deep cloning of complex object graphs with reference preservation - **Security**: Class registration and configurable deserialization policies +## Installation + +### Maven + +```xml + + org.apache.fory + fory-core + 1.1.0 + +``` + +### Gradle + +```kotlin +implementation("org.apache.fory:fory-core:1.1.0") +``` + +### JDK25+ + +On JDK25+, put Fory on the module path and open `java.lang.invoke` to the Fory +core module: + +```bash +--add-opens=java.base/java.lang.invoke=org.apache.fory.core +``` + ## Quick Start Note that Fory creation is not cheap, the **Fory instances should be reused between serializations** instead of creating it every time. You should keep Fory as a static global variable, or instance variable of some singleton object or limited objects. diff --git a/docs/guide/java/native-serialization.md b/docs/guide/java/native-serialization.md index 3129a8a9ec..fb75ad2bb3 100644 --- a/docs/guide/java/native-serialization.md +++ b/docs/guide/java/native-serialization.md @@ -165,19 +165,19 @@ path is too expensive. ## Final Fields And Constructors Records are deserialized through their canonical constructor. Ordinary classes use Fory's normal -object-creation path and field setting, including final fields when the runtime supports it. Fory -does not expose a constructor-mapping annotation or a constructor-registration API for ordinary -classes, and Java parameter-name metadata such as `-parameters` or `@ConstructorProperties` is not a -Fory object-creation contract. - -If an ordinary class cannot be created by supported Java mechanisms, use an accessible no-argument -constructor, model it as a record when canonical-constructor semantics are appropriate, or register a -custom serializer. On JDK25+ with Unsafe memory access denied, use the -`java.base/java.lang.invoke` open shown in troubleshooting for supported final-field and JDK access -paths. Fory does not require `--enable-final-field-mutation` for ordinary final-field restoration on -JDK26+. See -[Troubleshooting](troubleshooting.md#jdk25-zero-unsafe-mode-and-module-opens) for the required JVM -flags. +object-creation path and field setting, including final fields when the runtime supports it: + +```java +public final class User { + private final String name; + private final int age; + + public User(String name, int age) { + this.name = name; + this.age = age; + } +} +``` ## JDK Serialization Hooks diff --git a/docs/guide/java/troubleshooting.md b/docs/guide/java/troubleshooting.md index 394677d605..f0286c92aa 100644 --- a/docs/guide/java/troubleshooting.md +++ b/docs/guide/java/troubleshooting.md @@ -148,32 +148,16 @@ Fory fory = Fory.builder() fory.registerSerializer(MyClass.class, new MyClassSerializer(fory.getTypeResolver())); ``` -### JDK25+ zero-Unsafe mode and module opens +### JDK25+ access errors -When running on JDK25+ with Unsafe memory access denied, or on a later JDK where denied Unsafe -memory access becomes the default, start the JVM with: - -```bash ---sun-misc-unsafe-memory-access=deny -``` - -Run Fory as named modules on the module path and open `java.base/java.lang.invoke` to the Fory core -module: +On JDK25+, if an error names `java.base/java.lang.invoke`, put Fory on the module path and open +`java.lang.invoke` to the Fory core module: ```bash --add-opens=java.base/java.lang.invoke=org.apache.fory.core ``` -If this open is missing, Fory reports an error that names `java.base/java.lang.invoke`. - -Fory does not require `--enable-final-field-mutation` for normal final-field restoration on JDK26 -and later. With the `java.base/java.lang.invoke` open above, Fory uses trusted lookup field handles -instead of ordinary reflective final-field mutation. Named application modules that contain private -fields still need to open the application package to `org.apache.fory.core`. - -The vectorized Arrow APIs in `fory-format` depend on Apache Arrow's memory layer. With the current -Arrow dependency, those APIs are unavailable when `--sun-misc-unsafe-memory-access=deny` is set -because Arrow initializes its own `sun.misc.Unsafe` memory access internally. +Fory does not require application package opens for private-field access. ## Performance Issues diff --git a/docs/guide/kotlin/configuration.md b/docs/guide/kotlin/configuration.md index 23004d35b6..fc44d872ea 100644 --- a/docs/guide/kotlin/configuration.md +++ b/docs/guide/kotlin/configuration.md @@ -97,18 +97,6 @@ val fory: ThreadSafeFory = ForyKotlin.builder() All configuration options from Fory Java are available. See [Java Configuration](../java/configuration.md) for the complete list. -## JDK25+ Zero-Unsafe Mode - -On JDK25+ with Unsafe memory access denied, Kotlin classes follow the same final-field rules as Java -native serialization. Ordinary runtime serializers use Fory's supported object-creation path and -field setting; KSP-generated `@ForyStruct` serializers call source-visible primary constructors -directly. Fory does not expose constructor-mapping APIs for normal Kotlin classes. If a class cannot -be created by supported Java mechanisms, use an accessible no-argument constructor, a generated -`@ForyStruct` serializer, or a custom serializer. - -The JVM also needs the module opens listed in -[Java Troubleshooting](../java/troubleshooting.md#jdk25-zero-unsafe-mode-and-module-opens). - Common options for Kotlin native-mode payloads: ```kotlin diff --git a/docs/guide/kotlin/index.md b/docs/guide/kotlin/index.md index fb4ccffc40..76325897c7 100644 --- a/docs/guide/kotlin/index.md +++ b/docs/guide/kotlin/index.md @@ -62,6 +62,15 @@ See [Java Features](../java/index.md#features) for complete feature list. implementation("org.apache.fory:fory-kotlin:1.1.0") ``` +### JDK25+ + +Kotlin uses the Fory Java core at runtime. On JDK25+, put Fory on the module +path and open `java.lang.invoke` to the Fory core module: + +```bash +--add-opens=java.base/java.lang.invoke=org.apache.fory.core +``` + ## Quick Start ```kotlin diff --git a/docs/guide/kotlin/static-generated-serializers.md b/docs/guide/kotlin/static-generated-serializers.md index 0282c84781..97f8150e1e 100644 --- a/docs/guide/kotlin/static-generated-serializers.md +++ b/docs/guide/kotlin/static-generated-serializers.md @@ -60,8 +60,7 @@ import org.apache.fory.kotlin.Fixed import org.apache.fory.kotlin.VarInt @ForyStruct -data class User -constructor( +data class User( @ForyField(id = 1) val id: @Fixed UInt, @@ -82,10 +81,8 @@ them. The processor generates serializers for public or internal, concrete, non-generic classes in named packages. A supported class must have a primary -constructor whose serialized parameters are `val` or `var` properties with the -same names as the constructor parameters. `data class` is the common case, but it -is not required. Mutable no-argument structs can instead expose serialized `var` -properties with `@ForyField`. +constructor whose serialized parameters are `val` or `var` properties. `data +class` is the common case, but it is not required. Internal Kotlin struct classes are supported when KSP runs in the same Kotlin module that owns the struct. The generated Kotlin serializer is also internal, @@ -118,8 +115,7 @@ inside collections and maps. ```kotlin @ForyStruct -data class NullabilityExample -constructor( +data class NullabilityExample( @ForyField(id = 1) val a: List, diff --git a/docs/guide/scala/index.md b/docs/guide/scala/index.md index 2c925c2854..c861bb0894 100644 --- a/docs/guide/scala/index.md +++ b/docs/guide/scala/index.md @@ -52,6 +52,15 @@ Add the dependency with sbt: libraryDependencies += "org.apache.fory" %% "fory-scala" % "1.1.0" ``` +### JDK25+ + +Scala uses the Fory Java core at runtime. On JDK25+, put Fory on the module path +and open `java.lang.invoke` to the Fory core module: + +```bash +--add-opens=java.base/java.lang.invoke=org.apache.fory.core +``` + ## Quick Start ```scala diff --git a/java/fory-core/src/main/java/org/apache/fory/context/ReadContext.java b/java/fory-core/src/main/java/org/apache/fory/context/ReadContext.java index 79398e2293..6dca2e503a 100644 --- a/java/fory-core/src/main/java/org/apache/fory/context/ReadContext.java +++ b/java/fory-core/src/main/java/org/apache/fory/context/ReadContext.java @@ -30,7 +30,6 @@ import org.apache.fory.resolver.TypeInfo; import org.apache.fory.resolver.TypeInfoHolder; import org.apache.fory.resolver.TypeResolver; -import org.apache.fory.serializer.AbstractObjectSerializer; import org.apache.fory.serializer.PrimitiveSerializers.LongSerializer; import org.apache.fory.serializer.Serializer; import org.apache.fory.serializer.StringSerializer; @@ -343,12 +342,6 @@ public int preserveRefId(int refId) { /** Delegates to {@link RefReader#tryPreserveRefId(MemoryBuffer)} on the current buffer. */ public int tryPreserveRefId() { - if (refReader.hasPreservedRefId()) { - // Constructor-bound objects cannot satisfy self-references until construction finishes. - // The guard keeps ordinary ref reads on the direct path after standard serializers bind - // their object before reading fields. - AbstractObjectSerializer.trackConstructorRefRead(this, buffer); - } return refReader.tryPreserveRefId(buffer); } @@ -499,7 +492,7 @@ public String readString() { public String readStringRef() { MemoryBuffer buffer = this.buffer; if (stringSerializer.needToWriteRef()) { - int nextReadRefId = tryPreserveRefId(); + int nextReadRefId = refReader.tryPreserveRefId(buffer); if (nextReadRefId >= Fory.NOT_NULL_VALUE_FLAG) { String obj = stringSerializer.read(this); refReader.setReadRef(nextReadRefId, obj); @@ -533,7 +526,8 @@ public long readInt64() { * directly. */ public Object readRef() { - int nextReadRefId = tryPreserveRefId(); + MemoryBuffer buffer = this.buffer; + int nextReadRefId = refReader.tryPreserveRefId(buffer); if (nextReadRefId >= Fory.NOT_NULL_VALUE_FLAG) { TypeInfo typeInfo = typeResolver.readTypeInfo(this); Object o = readNonRef(typeInfo); @@ -545,7 +539,7 @@ public Object readRef() { /** Variant of {@link #readRef()} that uses already resolved {@link TypeInfo}. */ public Object readRef(TypeInfo typeInfo) { - int nextReadRefId = tryPreserveRefId(); + int nextReadRefId = refReader.tryPreserveRefId(buffer); if (nextReadRefId >= Fory.NOT_NULL_VALUE_FLAG) { Object o = readNonRef(typeInfo); refReader.setReadRef(nextReadRefId, o); @@ -556,7 +550,7 @@ public Object readRef(TypeInfo typeInfo) { /** Variant of {@link #readRef()} that reuses a cached type-info holder. */ public Object readRef(TypeInfoHolder classInfoHolder) { - int nextReadRefId = tryPreserveRefId(); + int nextReadRefId = refReader.tryPreserveRefId(buffer); if (nextReadRefId >= Fory.NOT_NULL_VALUE_FLAG) { TypeInfo typeInfo = typeResolver.readTypeInfo(this, classInfoHolder); Object o = readNonRef(typeInfo); @@ -569,7 +563,7 @@ public Object readRef(TypeInfoHolder classInfoHolder) { /** Reads a nullable object using an already chosen serializer. */ public T readRef(Serializer serializer) { if (serializer.needToWriteRef()) { - int nextReadRefId = tryPreserveRefId(); + int nextReadRefId = refReader.tryPreserveRefId(buffer); if (nextReadRefId >= Fory.NOT_NULL_VALUE_FLAG) { Object o = readNonRef(serializer); refReader.setReadRef(nextReadRefId, o); diff --git a/java/fory-core/src/main/java/org/apache/fory/memory/MemoryUtils.java b/java/fory-core/src/main/java/org/apache/fory/memory/MemoryUtils.java index bbcc892205..7f7b06c27f 100644 --- a/java/fory-core/src/main/java/org/apache/fory/memory/MemoryUtils.java +++ b/java/fory-core/src/main/java/org/apache/fory/memory/MemoryUtils.java @@ -21,34 +21,22 @@ import java.nio.ByteBuffer; import org.apache.fory.platform.AndroidSupport; -import org.apache.fory.platform.GraalvmSupport; import org.apache.fory.platform.internal._JDKAccess; /** Memory utils for fory. */ public class MemoryUtils { - // JDK25+ internal-field access must be backed by supported access in the multi-release classes. - // The JDK25+ replacement obtains a trusted lookup through java.base/java.lang.invoke instead of - // requiring per-package JDK opens or jdk.unsupported. + // Android does not expose these private JDK fields. GraalVM native image support is decided by + // the accessor owner instead of being disabled globally here. public static final boolean JDK_INTERNAL_FIELD_ACCESS = - !AndroidSupport.IS_ANDROID - && !GraalvmSupport.IN_GRAALVM_NATIVE_IMAGE - && _JDKAccess.JDK_INTERNAL_FIELD_ACCESS; + !AndroidSupport.IS_ANDROID && _JDKAccess.JDK_INTERNAL_FIELD_ACCESS; public static final boolean JDK_LANG_FIELD_ACCESS = - !AndroidSupport.IS_ANDROID - && !GraalvmSupport.IN_GRAALVM_NATIVE_IMAGE - && _JDKAccess.JDK_LANG_FIELD_ACCESS; + !AndroidSupport.IS_ANDROID && _JDKAccess.JDK_LANG_FIELD_ACCESS; public static final boolean JDK_COLLECTION_FIELD_ACCESS = - !AndroidSupport.IS_ANDROID - && !GraalvmSupport.IN_GRAALVM_NATIVE_IMAGE - && _JDKAccess.JDK_COLLECTION_FIELD_ACCESS; + !AndroidSupport.IS_ANDROID && _JDKAccess.JDK_COLLECTION_FIELD_ACCESS; public static final boolean JDK_CONCURRENT_FIELD_ACCESS = - !AndroidSupport.IS_ANDROID - && !GraalvmSupport.IN_GRAALVM_NATIVE_IMAGE - && _JDKAccess.JDK_CONCURRENT_FIELD_ACCESS; + !AndroidSupport.IS_ANDROID && _JDKAccess.JDK_CONCURRENT_FIELD_ACCESS; public static final boolean JDK_PROXY_FIELD_ACCESS = - !AndroidSupport.IS_ANDROID - && !GraalvmSupport.IN_GRAALVM_NATIVE_IMAGE - && _JDKAccess.JDK_PROXY_FIELD_ACCESS; + !AndroidSupport.IS_ANDROID && _JDKAccess.JDK_PROXY_FIELD_ACCESS; public static MemoryBuffer buffer(int size) { return wrap(new byte[size]); diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/AbstractObjectSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/AbstractObjectSerializer.java index 1bfd2a5e7c..5312540d41 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/AbstractObjectSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/AbstractObjectSerializer.java @@ -70,9 +70,9 @@ public abstract class AbstractObjectSerializer extends Serializer { private static final Logger LOG = LoggerFactory.getLogger(AbstractObjectSerializer.class); private static final Object SELF_REFERENCE = new Object(); // Constructor-bound objects reserve a ref id before constructor arguments are read, but the - // object cannot be referenced semantically until the constructor returns. ReadContext calls the - // tracker from tryPreserveRefId so nested collection/map/array elements cannot hide an unresolved - // self-reference inside a constructor argument. + // object cannot be referenced semantically until the constructor returns. Generated constructor + // serializers call the tracker before reading ref-tracking constructor-phase fields so nested + // collection/map/array elements cannot hide unresolved self-references. private static final Object CONSTRUCTOR_REF_IDS = new Object(); private static final Object UNRESOLVED_CONSTRUCTOR_REF_IDS = new Object(); protected final Config config; @@ -1083,7 +1083,15 @@ private SerializationFieldInfo[] buildFieldsInfo() { } protected T newBean() { - return objectInstantiator.newInstance(); + ObjectInstantiator instantiator = objectInstantiator; + if (instantiator == null) { + throw objectCreationUnsupported(); + } + return instantiator.newInstance(); + } + + private ForyException objectCreationUnsupported() { + return new ForyException("Serializer for " + type.getName() + " does not create objects"); } protected final void checkNoUnresolvedReadRef(ReadContext readContext) { diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/CompatibleLayerSerializerBase.java b/java/fory-core/src/main/java/org/apache/fory/serializer/CompatibleLayerSerializerBase.java index b696a21a67..e7040f16f4 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/CompatibleLayerSerializerBase.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/CompatibleLayerSerializerBase.java @@ -30,7 +30,6 @@ import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.meta.TypeDef; import org.apache.fory.reflect.FieldAccessor; -import org.apache.fory.reflect.ObjectInstantiator; import org.apache.fory.resolver.TypeResolver; import org.apache.fory.serializer.FieldGroups.SerializationFieldInfo; import org.apache.fory.type.DescriptorGrouper; @@ -43,37 +42,14 @@ */ @SuppressWarnings({"unchecked", "rawtypes"}) public abstract class CompatibleLayerSerializerBase extends AbstractObjectSerializer { - // Layer serializers are field-only views over an already-created subclass instance. A real - // object instantiator would move construction ownership into each superclass slot serializer. - private static final ObjectInstantiator FIELD_ONLY_INSTANTIATOR = - new FieldOnlyInstantiator(); - - private static final class FieldOnlyInstantiator extends ObjectInstantiator { - private FieldOnlyInstantiator() { - super(Object.class); - } - - @Override - public Object newInstance() { - throw new UnsupportedOperationException("Layer serializers do not create objects"); - } - - @Override - public Object newInstanceWithArguments(Object... arguments) { - throw new UnsupportedOperationException("Layer serializers do not create objects"); - } - } - protected TypeDef layerTypeDef; protected Class layerMarkerClass; protected SerializationFieldInfo[] allFields = new SerializationFieldInfo[0]; public CompatibleLayerSerializerBase(TypeResolver typeResolver, Class type) { - super(typeResolver, type, fieldOnlyInstantiator()); - } - - private static ObjectInstantiator fieldOnlyInstantiator() { - return (ObjectInstantiator) FIELD_ONLY_INSTANTIATOR; + // Layer serializers are field-only views over an already-created subclass instance. Null keeps + // object construction with the concrete serializer owner. + super(typeResolver, type, null); } public final void setLayerSerializerMeta(TypeDef layerTypeDef, Class layerMarkerClass) { diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/ExceptionSerializers.java b/java/fory-core/src/main/java/org/apache/fory/serializer/ExceptionSerializers.java index c13b52cc85..ae23e3469b 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/ExceptionSerializers.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/ExceptionSerializers.java @@ -54,29 +54,9 @@ @SuppressWarnings({"rawtypes", "unchecked"}) public final class ExceptionSerializers { private static final Set> THROWABLE_SUPER_CLASSES = ofHashSet(Throwable.class); - // Throwable slot serializers populate fields on the throwable allocated by ExceptionSerializer. - // They must not resolve constructors for each serialized superclass layer. - private static final ObjectInstantiator FIELD_ONLY_INSTANTIATOR = - new FieldOnlyInstantiator(); private ExceptionSerializers() {} - private static final class FieldOnlyInstantiator extends ObjectInstantiator { - private FieldOnlyInstantiator() { - super(Object.class); - } - - @Override - public Object newInstance() { - throw new UnsupportedOperationException("Throwable layer serializers do not create objects"); - } - - @Override - public Object newInstanceWithArguments(Object... arguments) { - throw new UnsupportedOperationException("Throwable layer serializers do not create objects"); - } - } - public static final class ExceptionSerializer extends Serializer { private final Config config; private final TypeResolver typeResolver; @@ -444,8 +424,9 @@ private static Serializer[] buildSlotsSerializers(TypeResolver typeResolver, slotsSerializer = new CompatibleLayerSerializer(typeResolver, type, layerTypeDef, layerMarkerClass); } else { - slotsSerializer = - new ObjectSerializer<>(typeResolver, type, false, fieldOnlyInstantiator()); + // Throwable slot serializers populate fields on the throwable allocated by + // ExceptionSerializer. + slotsSerializer = new ObjectSerializer<>(typeResolver, type, false, null); } serializers.add(slotsSerializer); type = (Class) type.getSuperclass(); @@ -455,10 +436,6 @@ private static Serializer[] buildSlotsSerializers(TypeResolver typeResolver, return serializers.toArray(new Serializer[0]); } - private static ObjectInstantiator fieldOnlyInstantiator() { - return (ObjectInstantiator) FIELD_ONLY_INSTANTIATOR; - } - private static void readAndSetFields( ReadContext readContext, Object target, Serializer[] slotsSerializers, Config config) { readAndCheckNumClassLayers(readContext, target.getClass(), slotsSerializers.length); diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/StaticGeneratedStructSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/StaticGeneratedStructSerializer.java index 886cd613f8..81091c85d7 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/StaticGeneratedStructSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/StaticGeneratedStructSerializer.java @@ -20,8 +20,6 @@ package org.apache.fory.serializer; import java.lang.reflect.Constructor; -import java.lang.reflect.Field; -import java.lang.reflect.Modifier; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -45,7 +43,6 @@ import org.apache.fory.serializer.FieldGroups.SerializationFieldInfo; import org.apache.fory.serializer.converter.FieldConverters; import org.apache.fory.type.Descriptor; -import org.apache.fory.type.DescriptorBuilder; import org.apache.fory.type.DescriptorGrouper; import org.apache.fory.util.StringUtils; @@ -175,25 +172,7 @@ public final FieldGroups buildLocalFieldGroups(List descriptors) { } protected final List runtimeDescriptors(List descriptors) { - return typeResolver.normalizeFieldDescriptors(type, true, attachFields(descriptors)); - } - - private List attachFields(List descriptors) { - Map fields = new HashMap<>(); - for (Field field : Descriptor.getFields(type)) { - fields.put(field.getDeclaringClass().getName() + "." + field.getName(), field); - } - List result = new ArrayList<>(descriptors.size()); - for (Descriptor descriptor : descriptors) { - if (descriptor.getField() != null || !Modifier.isFinal(descriptor.getModifier())) { - result.add(descriptor); - continue; - } - Field field = fields.get(fieldKey(descriptor)); - result.add( - field == null ? descriptor : new DescriptorBuilder(descriptor).field(field).build()); - } - return result; + return typeResolver.normalizeFieldDescriptors(type, true, descriptors); } private static boolean hasSourceOnlyMetadata(List descriptors) { @@ -294,14 +273,6 @@ protected final void setGeneratedFieldValue( throw new ForyException("Generated field " + fieldInfo.getName() + " is not writable"); } - protected final Object copyConstructorFieldValue( - CopyContext copyContext, - Object originObject, - Object fieldValue, - SerializationFieldInfo fieldInfo) { - return copyFieldValue(copyContext, fieldValue, fieldInfo); - } - protected final void writeBuildInFieldValue( WriteContext writeContext, SerializationFieldInfo fieldInfo, Object fieldValue) { // Some schema-built-in fields still use container-shaped Java accessors, such as diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/ChildContainerSerializers.java b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/ChildContainerSerializers.java index 25f16464c0..b3daa780a3 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/ChildContainerSerializers.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/ChildContainerSerializers.java @@ -52,7 +52,6 @@ import org.apache.fory.exception.ForyException; import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.meta.TypeDef; -import org.apache.fory.reflect.ObjectInstantiator; import org.apache.fory.reflect.ReflectionUtils; import org.apache.fory.resolver.ClassResolver; import org.apache.fory.resolver.TypeResolver; @@ -74,27 +73,6 @@ */ @SuppressWarnings({"unchecked", "rawtypes"}) public class ChildContainerSerializers { - // Child-container slot serializers populate fields on the container instance created by the - // concrete container serializer. Parent slots do not own object construction. - private static final ObjectInstantiator FIELD_ONLY_INSTANTIATOR = - new FieldOnlyInstantiator(); - - private static final class FieldOnlyInstantiator extends ObjectInstantiator { - private FieldOnlyInstantiator() { - super(Object.class); - } - - @Override - public Object newInstance() { - throw new UnsupportedOperationException("Child-container slots do not create objects"); - } - - @Override - public Object newInstanceWithArguments(Object... arguments) { - throw new UnsupportedOperationException("Child-container slots do not create objects"); - } - } - public static Class getCollectionSerializerClass(Class cls) { if (ChildCollectionSerializer.superClasses.contains(cls) || ChildSortedSetSerializer.superClasses.contains(cls) @@ -647,7 +625,9 @@ private static Serializer[] buildSlotsSerializers( slotsSerializer = new CompatibleLayerSerializer(typeResolver, cls, layerTypeDef, layerMarkerClass); } else { - slotsSerializer = new ObjectSerializer<>(typeResolver, cls, false, slotInstantiator()); + // Slot serializers only populate fields on the container instance created by the concrete + // container serializer. + slotsSerializer = new ObjectSerializer<>(typeResolver, cls, false, null); } serializers.add(slotsSerializer); cls = (Class) cls.getSuperclass(); @@ -657,10 +637,6 @@ private static Serializer[] buildSlotsSerializers( return serializers.toArray(new Serializer[0]); } - private static ObjectInstantiator slotInstantiator() { - return (ObjectInstantiator) FIELD_ONLY_INSTANTIATOR; - } - private static void readAndSetFields( ReadContext readContext, TypeResolver typeResolver, diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/CollectionSerializers.java b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/CollectionSerializers.java index 72758e6598..e1ea4b0217 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/CollectionSerializers.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/CollectionSerializers.java @@ -57,7 +57,6 @@ import org.apache.fory.exception.ForyException; import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.memory.MemoryUtils; -import org.apache.fory.platform.GraalvmSupport; import org.apache.fory.platform.JdkVersion; import org.apache.fory.reflect.FieldAccessor; import org.apache.fory.reflect.ReflectionUtils; @@ -123,6 +122,15 @@ private static void checkBoundedQueueCapacity(Config config, int numElements, in } } + private static UnsupportedOperationException unsupportedBoundedQueueWrite(Class type) { + return new UnsupportedOperationException( + "Serializing or copying " + + type.getName() + + " requires access to its exact capacity field. This runtime can deserialize existing " + + "payloads for this type, but cannot serialize or copy it without JDK concurrent " + + "field access."); + } + public static final class ArrayListSerializer extends CollectionSerializer { public ArrayListSerializer(TypeResolver typeResolver) { super(typeResolver, ArrayList.class, true); @@ -172,7 +180,7 @@ public void write(WriteContext writeContext, List value) { super.write(writeContext, value); } else { Object[] array = - !MemoryUtils.JDK_COLLECTION_FIELD_ACCESS || GraalvmSupport.IN_GRAALVM_NATIVE_IMAGE + !MemoryUtils.JDK_COLLECTION_FIELD_ACCESS ? value.toArray() : (Object[]) ArrayAccess.ACCESSOR.getObject(value); writeContext.writeRef(array); @@ -617,7 +625,7 @@ public Collection newCollection(ReadContext readContext) { set = Collections.newSetFromMap(mapSerializer.newMap(readContext)); setNumElements(mapSerializer.getAndClearNumElements()); } else { - if (!MemoryUtils.JDK_COLLECTION_FIELD_ACCESS || GraalvmSupport.IN_GRAALVM_NATIVE_IMAGE) { + if (!MemoryUtils.JDK_COLLECTION_FIELD_ACCESS) { throw new UnsupportedOperationException( "This runtime cannot read legacy SetFromMap payloads that require hidden JDK field " + "restoration"); @@ -640,7 +648,7 @@ public Collection newCollection(ReadContext readContext) { @Override public Collection newCollection(CopyContext copyContext, Collection originCollection) { assert !config.isXlang(); - if (!MemoryUtils.JDK_COLLECTION_FIELD_ACCESS || GraalvmSupport.IN_GRAALVM_NATIVE_IMAGE) { + if (!MemoryUtils.JDK_COLLECTION_FIELD_ACCESS) { if (JdkVersion.MAJOR_VERSION >= 25) { throw unsupportedJdk25SetFromMap(null); } @@ -662,7 +670,7 @@ public Collection onCollectionWrite(WriteContext writeContext, Set value) { MemoryBuffer buffer = writeContext.getBuffer(); Map map; TypeInfo typeInfo; - if (!MemoryUtils.JDK_COLLECTION_FIELD_ACCESS || GraalvmSupport.IN_GRAALVM_NATIVE_IMAGE) { + if (!MemoryUtils.JDK_COLLECTION_FIELD_ACCESS) { if (JdkVersion.MAJOR_VERSION >= 25) { throw unsupportedJdk25SetFromMap(null); } @@ -932,8 +940,8 @@ public ArrayBlockingQueueSerializer(TypeResolver typeResolver, Class set = new TreeSet<>( - (s1, s2) -> { - int delta = s1.length() - s2.length(); - if (delta == 0) { - return s1.compareTo(s2); - } else { - return delta; - } - }); + (Comparator & Serializable) + (s1, s2) -> { + int delta = s1.length() - s2.length(); + if (delta == 0) { + return s1.compareTo(s2); + } else { + return delta; + } + }); set.add("str11"); set.add("str2"); assertEquals(set, serDe(fory, set)); diff --git a/kotlin/fory-kotlin-ksp/src/main/kotlin/org/apache/fory/kotlin/ksp/KotlinSerializerSourceWriter.kt b/kotlin/fory-kotlin-ksp/src/main/kotlin/org/apache/fory/kotlin/ksp/KotlinSerializerSourceWriter.kt index ae784daf34..2d2980069d 100644 --- a/kotlin/fory-kotlin-ksp/src/main/kotlin/org/apache/fory/kotlin/ksp/KotlinSerializerSourceWriter.kt +++ b/kotlin/fory-kotlin-ksp/src/main/kotlin/org/apache/fory/kotlin/ksp/KotlinSerializerSourceWriter.kt @@ -362,6 +362,7 @@ internal class KotlinSerializerSourceWriter(private val struct: KotlinSourceStru builder.append( " private fun readCompatibleConstructorField(readContext: ReadContext, remoteField: StaticGeneratedStructSerializer.RemoteFieldInfo, localField: SerializationFieldInfo, fieldId: Int): Any? {\n" ) + builder.append(" val buffer = readContext.buffer\n") builder.append(" return when (fieldId) {\n") for (field in struct.fields) { val readExpression = @@ -371,7 +372,14 @@ internal class KotlinSerializerSourceWriter(private val struct: KotlinSourceStru compatible = true, ) val expression = constructorReadExpression(field, readExpression) - builder.append(" ").append(field.id).append(" -> ").append(expression).append("\n") + if (field.trackingRef) { + builder.append(" ").append(field.id).append(" -> {\n") + builder.append(" trackConstructorRefRead(readContext, buffer)\n") + builder.append(" ").append(expression).append("\n") + builder.append(" }\n") + } else { + builder.append(" ").append(field.id).append(" -> ").append(expression).append("\n") + } } builder.append( " else -> throw IllegalStateException(\"Unknown generated field id \${fieldId}\")\n" @@ -550,7 +558,14 @@ internal class KotlinSerializerSourceWriter(private val struct: KotlinSourceStru val readExpression = direct ?: castReadExpression(field, "readFieldValue(readContext, fieldInfo)") val expression = constructorReadExpression(field, readExpression) - builder.append(" ").append(field.id).append(" -> ").append(expression).append("\n") + if (field.trackingRef) { + builder.append(" ").append(field.id).append(" -> {\n") + builder.append(" trackConstructorRefRead(readContext, buffer)\n") + builder.append(" ").append(expression).append("\n") + builder.append(" }\n") + } else { + builder.append(" ").append(field.id).append(" -> ").append(expression).append("\n") + } } builder.append( " else -> throw IllegalStateException(\"Unknown generated field id \${fieldId}\")\n" @@ -833,8 +848,13 @@ internal class KotlinSerializerSourceWriter(private val struct: KotlinSourceStru builder.append(indent).append(" if (canReadRemoteField(remoteField, localField)) {\n") val readExpression = "readCompatibleFieldValue(readContext, remoteField, localField)" val constructorReadExpression = - if (constructorRefs) "ctorFieldValue(readContext, $readExpression, type)" - else readExpression + if (constructorRefs && field.trackingRef) { + "run { trackConstructorRefRead(readContext, readContext.buffer); ctorFieldValue(readContext, $readExpression, type) }" + } else if (constructorRefs) { + "ctorFieldValue(readContext, $readExpression, type)" + } else { + readExpression + } builder .append(indent) .append(" ") @@ -1049,8 +1069,7 @@ internal class KotlinSerializerSourceWriter(private val struct: KotlinSourceStru if (field.nullable) "value.${field.name}?.copyOf()" else "value.${field.name}.copyOf()" field.type.isCollectionOrMap() -> return copyContainerExpression(field.type, "value.${field.name}", 0) - else -> - "copyConstructorFieldValue(copyContext, value, value.${field.name}, fieldsById[${field.id}]!!)" + else -> "copyFieldValue(copyContext, value.${field.name}, fieldsById[${field.id}]!!)" } return constructorReadExpression(field, expression) } diff --git a/kotlin/fory-kotlin-ksp/src/test/kotlin/org/apache/fory/kotlin/ksp/ProcessorValidationTest.kt b/kotlin/fory-kotlin-ksp/src/test/kotlin/org/apache/fory/kotlin/ksp/ProcessorValidationTest.kt index 48a5ea22ac..9213512db3 100644 --- a/kotlin/fory-kotlin-ksp/src/test/kotlin/org/apache/fory/kotlin/ksp/ProcessorValidationTest.kt +++ b/kotlin/fory-kotlin-ksp/src/test/kotlin/org/apache/fory/kotlin/ksp/ProcessorValidationTest.kt @@ -125,9 +125,58 @@ class ProcessorValidationTest { assertTrue(source.contains("return User(userName = (fieldValues[0] as String))")) assertTrue(!source.contains("return User(name = field0!!)")) assertTrue(source.contains("fieldValues[0] = value.name")) - assertFalse(source.contains("copyConstructorFieldValue(copyContext, value, value.name")) assertFalse(source.contains("NATURAL_ORDER_COMPARATOR")) assertFalse(source.contains("requireXlangNaturalOrdering")) + assertFalse(source.contains("trackConstructorRefRead(readContext, buffer)")) + } + + @Test + fun tracksCtorRefs() { + val childType = + KotlinSourceTypeNode( + rawClassExpression = "Child::class.java", + kotlinTypeName = "example.Child", + valueTypeName = "Child", + typeName = "example.Child", + typeId = null, + nullable = false, + trackingRef = true, + primitive = false, + unsigned = false, + ) + val source = + KotlinSerializerSourceWriter( + KotlinSourceStruct( + packageName = "example", + typeName = "Node", + qualifiedTypeName = "example.Node", + serializerName = "Node_ForySerializer", + serializerVisibility = KotlinSerializerVisibility.PUBLIC, + fields = + listOf( + KotlinSourceField( + id = 0, + name = "child", + type = childType, + hasForyField = true, + foryFieldId = 1, + trackingRef = true, + dynamic = "AUTO", + arrayType = false, + hasDefault = false, + nullable = false, + propertyTypeName = "Child", + constructorParameterName = "child", + ) + ), + originatingFiles = emptyList(), + ) + ) + .write() + + assertTrue(source.contains("private fun readSchemaConstructorField")) + assertTrue(source.contains("private fun readCompatibleConstructorField")) + assertTrue(source.contains("trackConstructorRefRead(readContext, buffer)")) } @Test @@ -272,13 +321,8 @@ class ProcessorValidationTest { for (field in fields.take(6)) { assertTrue(source.contains("fieldValues[${field.id}] = value.${field.name}")) - assertFalse( - source.contains("copyConstructorFieldValue(copyContext, value, value.${field.name}") - ) } - assertTrue( - source.contains("fieldValues[6] = copyConstructorFieldValue(copyContext, value, value.child") - ) + assertTrue(source.contains("fieldValues[6] = copyFieldValue(copyContext, value.child")) } @Test @@ -404,7 +448,7 @@ class ProcessorValidationTest { type = mapType, hasForyField = true, foryFieldId = 1, - trackingRef = false, + trackingRef = true, dynamic = "AUTO", arrayType = false, hasDefault = false, @@ -470,9 +514,10 @@ class ProcessorValidationTest { ) assertTrue( source.contains( - "0 -> KotlinCollectionAdapters.toTreeMap((readFieldValue(readContext, fieldInfo) as kotlin.collections.Map))" + "KotlinCollectionAdapters.toTreeMap((readFieldValue(readContext, fieldInfo) as kotlin.collections.Map))" ) ) + assertTrue(source.contains("0 -> {\n trackConstructorRefRead(readContext, buffer)")) assertTrue( source.contains( "1 -> run { val readSource0 = ((readFieldValue(readContext, fieldInfo) as kotlin.collections.List>) as Collection<*>);" @@ -480,7 +525,7 @@ class ProcessorValidationTest { ) assertTrue( source.contains( - "0 -> KotlinCollectionAdapters.toTreeMap((readCompatibleFieldValue(readContext, remoteField, localField) as kotlin.collections.Map))" + "KotlinCollectionAdapters.toTreeMap((readCompatibleFieldValue(readContext, remoteField, localField) as kotlin.collections.Map))" ) ) assertTrue( @@ -521,7 +566,6 @@ class ProcessorValidationTest { ) assertTrue(source.contains("fieldValues[2] = run { val copySource0 = value.arrays;")) assertTrue(source.contains("copyTarget0.add(copyElement0.copyOf())")) - assertFalse(source.contains("fieldValues[0] = copyConstructorFieldValue")) assertFalse(source.contains("fieldValues[0] = KotlinCollectionAdapters.toTreeMap(run")) assertFalse(source.contains("java.util.TreeMap((copySource0.comparator()")) } @@ -536,7 +580,7 @@ class ProcessorValidationTest { typeName = "java.lang.String", typeId = "Types.STRING", nullable = false, - trackingRef = false, + trackingRef = true, primitive = false, unsigned = false, ) @@ -556,7 +600,7 @@ class ProcessorValidationTest { type = stringType, hasForyField = true, foryFieldId = 1, - trackingRef = false, + trackingRef = true, dynamic = "AUTO", arrayType = false, hasDefault = true, @@ -577,6 +621,9 @@ class ProcessorValidationTest { assertTrue(compatibleSource.contains("return readCompatibleDefaultConstructor(readContext)")) assertTrue(compatibleSource.contains("beginConstructorRef(readContext)")) assertTrue(compatibleSource.contains("checkNoUnresolvedReadRef(readContext)")) + assertTrue( + compatibleSource.contains("trackConstructorRefRead(readContext, readContext.buffer)") + ) assertTrue(compatibleSource.contains("referenceConstructorRef(readContext, constructed)")) assertTrue(compatibleSource.contains("endConstructorRef(readContext)")) assertTrue(compatibleSource.contains("missingDefaultMask")) From 97ab18a0d782f7f783cad5011f9a6c10315bdd66 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Wed, 3 Jun 2026 01:07:04 +0800 Subject: [PATCH 64/69] refactor(java): unify primitive object codec paths --- .agents/languages/java.md | 4 + .../fory/builder/ObjectCodecBuilder.java | 1589 +++++++---------- .../org/apache/fory/memory/MemoryBuffer.java | 2 + 3 files changed, 698 insertions(+), 897 deletions(-) diff --git a/.agents/languages/java.md b/.agents/languages/java.md index 8910192b8c..aa21bfaf7a 100644 --- a/.agents/languages/java.md +++ b/.agents/languages/java.md @@ -110,6 +110,10 @@ Load this file when changing anything under `java/` or when Java drives a cross- - Root codegen and builder classes that still need Unsafe on JDK8-24 must route symbolic Unsafe access through a helper with a Java 25 replacement. Do not leave `_JDKAccess.unsafe()` or `sun.misc.Unsafe` references in JDK25-visible classes outside matching `java25` replacements. +- `ObjectCodecBuilder` primitive generated-code paths must keep one primitive field traversal and + dispatch owner. Select direct Unsafe `(base, absoluteAddress)` versus indexed `MemoryBuffer` + `(buffer, intIndex)` access at codegen setup time; do not duplicate the primitive switch or emit a + per-field JDK-version branch. JDK25+ generated serializers must not reference `sun.misc.Unsafe`. - `_UnsafeUtils` is the JDK8-24 Unsafe owner only. It must have a Java25 replacement with no `sun.misc.Unsafe` fields, methods, constructors, imports, or descriptors, and MR-JAR plus benchmark checks must require that replacement so JDK25+ never links `jdk.unsupported` through diff --git a/java/fory-core/src/main/java/org/apache/fory/builder/ObjectCodecBuilder.java b/java/fory-core/src/main/java/org/apache/fory/builder/ObjectCodecBuilder.java index 0395d53bc5..41e5abb0c2 100644 --- a/java/fory-core/src/main/java/org/apache/fory/builder/ObjectCodecBuilder.java +++ b/java/fory-core/src/main/java/org/apache/fory/builder/ObjectCodecBuilder.java @@ -44,6 +44,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Set; import java.util.SortedMap; import java.util.TreeMap; import org.apache.fory.Fory; @@ -254,141 +255,79 @@ protected int getNumPrimitiveFields(List> primitiveGroups) { private List serializePrimitivesUnCompressed( Expression bean, Expression buffer, List> primitiveGroups, int totalSize) { - if (JdkVersion.MAJOR_VERSION >= 25) { - return serializeRawPrimitivesIndexed(bean, buffer, primitiveGroups, totalSize); - } List expressions = new ArrayList<>(); - int numPrimitiveFields = getNumPrimitiveFields(primitiveGroups); - Literal totalSizeLiteral = new Literal(totalSize, PRIMITIVE_INT_TYPE); - // After this grow, following writes can be unsafe without checks. + Literal totalSizeLiteral = Literal.ofInt(totalSize); + // After this grow, following writes can use unchecked low-level access. expressions.add(new Invoke(buffer, "grow", totalSizeLiteral)); - // Must grow first, otherwise may get invalid address. - Expression base = new Invoke(buffer, "getHeapMemory", "base", PRIMITIVE_BYTE_ARRAY_TYPE); - Expression writerAddr = - new Invoke(buffer, "_unsafeWriterAddress", "writerAddr", PRIMITIVE_LONG_TYPE); - expressions.add(base); - expressions.add(writerAddr); - int acc = 0; - for (List group : primitiveGroups) { - ListExpression groupExpressions = new ListExpression(); - // use Reference to cut-off expr dependency. - for (Descriptor descriptor : group) { - int dispatchId = getNumericDescriptorDispatchId(descriptor); - // `bean` will be replaced by `Reference` to cut-off expr dependency. - Expression fieldValue = getFieldValue(bean, descriptor); - if (fieldValue instanceof Inlineable) { - ((Inlineable) fieldValue).inline(); - } - if (dispatchId == DispatchId.BOOL) { - groupExpressions.add(unsafePutBoolean(base, getWriterPos(writerAddr, acc), fieldValue)); - acc += 1; - } else if (dispatchId == DispatchId.INT8) { - groupExpressions.add(unsafePut(base, getWriterPos(writerAddr, acc), fieldValue)); - acc += 1; - } else if (dispatchId == DispatchId.UINT8) { - groupExpressions.add( - unsafePut( - base, getWriterPos(writerAddr, acc), primitiveByteValue(fieldValue, descriptor))); - acc += 1; - } else if (dispatchId == DispatchId.CHAR) { - groupExpressions.add(unsafePutChar(base, getWriterPos(writerAddr, acc), fieldValue)); - acc += 2; - } else if (dispatchId == DispatchId.INT16) { - groupExpressions.add(unsafePutShort(base, getWriterPos(writerAddr, acc), fieldValue)); - acc += 2; - } else if (dispatchId == DispatchId.UINT16) { - groupExpressions.add( - unsafePutShort( - base, - getWriterPos(writerAddr, acc), - primitiveShortValue(fieldValue, descriptor))); - acc += 2; - } else if (dispatchId == DispatchId.FLOAT16 || dispatchId == DispatchId.BFLOAT16) { - groupExpressions.add( - unsafePutShort( - base, - getWriterPos(writerAddr, acc), - new Invoke(fieldValue, "toBits", SHORT_TYPE))); - acc += 2; - } else if (dispatchId == DispatchId.INT32) { - groupExpressions.add(unsafePutInt(base, getWriterPos(writerAddr, acc), fieldValue)); - acc += 4; - } else if (dispatchId == DispatchId.UINT32) { - groupExpressions.add( - unsafePutInt( - base, getWriterPos(writerAddr, acc), primitiveIntValue(fieldValue, descriptor))); - acc += 4; - } else if (dispatchId == DispatchId.INT64 || dispatchId == DispatchId.UINT64) { - groupExpressions.add(unsafePutLong(base, getWriterPos(writerAddr, acc), fieldValue)); - acc += 8; - } else if (dispatchId == DispatchId.FLOAT32) { - groupExpressions.add(unsafePutFloat(base, getWriterPos(writerAddr, acc), fieldValue)); - acc += 4; - } else if (dispatchId == DispatchId.FLOAT64) { - groupExpressions.add(unsafePutDouble(base, getWriterPos(writerAddr, acc), fieldValue)); - acc += 8; - } else { - throw new IllegalStateException("Unsupported dispatchId: " + dispatchId); - } - } - if (hasFewFields() || numPrimitiveFields < 4) { - expressions.add(groupExpressions); - } else { - expressions.add( - objectCodecOptimizer.invokeGenerated( - ofHashSet(bean, base, writerAddr), groupExpressions, "writeFields")); - } + PrimitiveWriteAccess access; + if (useIndexedAccess()) { + Expression writerIndex = new Invoke(buffer, "writerIndex", "writerIndex", PRIMITIVE_INT_TYPE); + expressions.add(writerIndex); + access = new BufferWriteAccess(buffer, writerIndex); + } else { + // Must grow first, otherwise may get invalid address. + Expression base = new Invoke(buffer, "getHeapMemory", "base", PRIMITIVE_BYTE_ARRAY_TYPE); + Expression writerAddr = + new Invoke(buffer, "_unsafeWriterAddress", "writerAddr", PRIMITIVE_LONG_TYPE); + expressions.add(base); + expressions.add(writerAddr); + access = new UnsafeWriteAccess(buffer, base, writerAddr); } - Expression increaseWriterIndex = - new Invoke( - buffer, - "_increaseWriterIndexUnsafe", - new Literal(totalSizeLiteral, PRIMITIVE_INT_TYPE)); - expressions.add(increaseWriterIndex); + writePrimitiveGroups(expressions, bean, buffer, primitiveGroups, () -> access, false); + expressions.add(new Invoke(buffer, "_increaseWriterIndexUnsafe", totalSizeLiteral)); return expressions; } private List serializePrimitivesCompressed( Expression bean, Expression buffer, List> primitiveGroups, int totalSize) { - if (JdkVersion.MAJOR_VERSION >= 25) { - return serializeCompressedIndexed(bean, buffer, primitiveGroups, totalSize); - } List expressions = new ArrayList<>(); - // int/long may need extra one-byte for writing. - int extraSize = 0; - for (List group : primitiveGroups) { - for (Descriptor d : group) { - int id = getNumericDescriptorDispatchId(d); - if (id == DispatchId.INT32 - || id == DispatchId.VARINT32 - || id == DispatchId.VAR_UINT32 - || id == DispatchId.UINT32) { - // varint may be written as 5bytes, use 8bytes for written as long to reduce cost. - extraSize += 4; - } else if (id == DispatchId.INT64 - || id == DispatchId.VARINT64 - || id == DispatchId.TAGGED_INT64 - || id == DispatchId.VAR_UINT64 - || id == DispatchId.TAGGED_UINT64 - || id == DispatchId.UINT64) { - extraSize += 1; // long use 1~9 bytes. - } - } - } + // int/long may need extra bytes for compressed writing. + int extraSize = extraPrimitiveSize(primitiveGroups); int growSize = totalSize + extraSize; - // After this grow, following writes can be unsafe without checks. + // After this grow, following writes can use unchecked low-level access. expressions.add(new Invoke(buffer, "grow", Literal.ofInt(growSize))); - // Must grow first, otherwise may get invalid address. - Expression base = new Invoke(buffer, "getHeapMemory", "base", PRIMITIVE_BYTE_ARRAY_TYPE); - expressions.add(base); + if (useIndexedAccess()) { + writePrimitiveGroups( + expressions, + bean, + buffer, + primitiveGroups, + () -> + new BufferWriteAccess( + buffer, new Invoke(buffer, "writerIndex", "writerIndex", PRIMITIVE_INT_TYPE)), + true); + } else { + // Must grow first, otherwise may get invalid address. + Expression base = new Invoke(buffer, "getHeapMemory", "base", PRIMITIVE_BYTE_ARRAY_TYPE); + expressions.add(base); + writePrimitiveGroups( + expressions, + bean, + buffer, + primitiveGroups, + () -> + new UnsafeWriteAccess( + buffer, + base, + new Invoke(buffer, "_unsafeWriterAddress", "writerAddr", PRIMITIVE_LONG_TYPE)), + true); + } + return expressions; + } + + private void writePrimitiveGroups( + List expressions, + Expression bean, + Expression buffer, + List> primitiveGroups, + WriteAccessFactory accessFactory, + boolean compressed) { int numPrimitiveFields = getNumPrimitiveFields(primitiveGroups); + int rawAcc = 0; for (List group : primitiveGroups) { ListExpression groupExpressions = new ListExpression(); - Expression writerAddr = - new Invoke(buffer, "_unsafeWriterAddress", "writerAddr", PRIMITIVE_LONG_TYPE); - // use Reference to cut-off expr dependency. - int acc = 0; - boolean compressStarted = false; + PrimitiveWriteAccess access = accessFactory.get(); + PrimitiveWriteState state = new PrimitiveWriteState(compressed ? 0 : rawAcc); for (Descriptor descriptor : group) { int dispatchId = getNumericDescriptorDispatchId(descriptor); // `bean` will be replaced by `Reference` to cut-off expr dependency. @@ -396,210 +335,144 @@ private List serializePrimitivesCompressed( if (fieldValue instanceof Inlineable) { ((Inlineable) fieldValue).inline(); } - if (dispatchId == DispatchId.BOOL) { - groupExpressions.add(unsafePutBoolean(base, getWriterPos(writerAddr, acc), fieldValue)); - acc += 1; - } else if (dispatchId == DispatchId.INT8) { - groupExpressions.add(unsafePut(base, getWriterPos(writerAddr, acc), fieldValue)); - acc += 1; - } else if (dispatchId == DispatchId.UINT8) { - groupExpressions.add( - unsafePut( - base, getWriterPos(writerAddr, acc), primitiveByteValue(fieldValue, descriptor))); - acc += 1; - } else if (dispatchId == DispatchId.CHAR) { - groupExpressions.add(unsafePutChar(base, getWriterPos(writerAddr, acc), fieldValue)); - acc += 2; - } else if (dispatchId == DispatchId.INT16) { - groupExpressions.add(unsafePutShort(base, getWriterPos(writerAddr, acc), fieldValue)); - acc += 2; - } else if (dispatchId == DispatchId.UINT16) { - groupExpressions.add( - unsafePutShort( - base, - getWriterPos(writerAddr, acc), - primitiveShortValue(fieldValue, descriptor))); - acc += 2; - } else if (dispatchId == DispatchId.FLOAT16 || dispatchId == DispatchId.BFLOAT16) { - groupExpressions.add( - unsafePutShort( - base, - getWriterPos(writerAddr, acc), - new Invoke(fieldValue, "toBits", SHORT_TYPE))); - acc += 2; - } else if (dispatchId == DispatchId.FLOAT32) { - groupExpressions.add(unsafePutFloat(base, getWriterPos(writerAddr, acc), fieldValue)); - acc += 4; - } else if (dispatchId == DispatchId.FLOAT64) { - groupExpressions.add(unsafePutDouble(base, getWriterPos(writerAddr, acc), fieldValue)); - acc += 8; - } else if (dispatchId == DispatchId.INT32) { - groupExpressions.add(unsafePutInt(base, getWriterPos(writerAddr, acc), fieldValue)); - acc += 4; - } else if (dispatchId == DispatchId.UINT32) { - groupExpressions.add( - unsafePutInt( - base, getWriterPos(writerAddr, acc), primitiveIntValue(fieldValue, descriptor))); - acc += 4; - } else if (dispatchId == DispatchId.INT64 || dispatchId == DispatchId.UINT64) { - groupExpressions.add(unsafePutLong(base, getWriterPos(writerAddr, acc), fieldValue)); - acc += 8; - } else if (dispatchId == DispatchId.VARINT32) { - if (!compressStarted) { - addIncWriterIndexExpr(groupExpressions, buffer, acc); - compressStarted = true; - } - groupExpressions.add(new Invoke(buffer, "_unsafeWriteVarInt32", fieldValue)); - } else if (dispatchId == DispatchId.VAR_UINT32) { - if (!compressStarted) { - addIncWriterIndexExpr(groupExpressions, buffer, acc); - compressStarted = true; - } - groupExpressions.add( - new Invoke( - buffer, "_unsafeWriteVarUInt32", primitiveIntValue(fieldValue, descriptor))); - } else if (dispatchId == DispatchId.VARINT64) { - if (!compressStarted) { - addIncWriterIndexExpr(groupExpressions, buffer, acc); - compressStarted = true; - } - groupExpressions.add(new Invoke(buffer, "writeVarInt64", fieldValue)); - } else if (dispatchId == DispatchId.TAGGED_INT64) { - if (!compressStarted) { - addIncWriterIndexExpr(groupExpressions, buffer, acc); - compressStarted = true; - } - groupExpressions.add(new Invoke(buffer, "writeTaggedInt64", fieldValue)); - } else if (dispatchId == DispatchId.VAR_UINT64) { - if (!compressStarted) { - addIncWriterIndexExpr(groupExpressions, buffer, acc); - compressStarted = true; - } - groupExpressions.add(new Invoke(buffer, "writeVarUInt64", fieldValue)); - } else if (dispatchId == DispatchId.TAGGED_UINT64) { - if (!compressStarted) { - addIncWriterIndexExpr(groupExpressions, buffer, acc); - compressStarted = true; - } - groupExpressions.add(new Invoke(buffer, "writeTaggedUInt64", fieldValue)); + if (compressed) { + writeCompressed( + groupExpressions, buffer, access, descriptor, dispatchId, fieldValue, state); } else { - throw new IllegalStateException("Unsupported dispatchId: " + dispatchId); + writeFixed(groupExpressions, access, descriptor, dispatchId, fieldValue, state); } } - if (!compressStarted) { - // int/long are sorted in the last. - addIncWriterIndexExpr(groupExpressions, buffer, acc); + if (compressed) { + if (!state.compressStarted) { + // int/long are sorted in the last. + addIncWriterIndexExpr(groupExpressions, buffer, state.acc); + } + } else { + rawAcc = state.acc; } if (hasFewFields() || numPrimitiveFields < 4) { expressions.add(groupExpressions); } else { expressions.add( objectCodecOptimizer.invokeGenerated( - ofHashSet(bean, buffer, base), groupExpressions, "writeFields")); + compressed ? access.compressedScope(bean) : access.fixedScope(bean), + groupExpressions, + "writeFields")); } } - return expressions; } - private List serializeRawPrimitivesIndexed( - Expression bean, Expression buffer, List> primitiveGroups, int totalSize) { - List expressions = new ArrayList<>(); - int numPrimitiveFields = getNumPrimitiveFields(primitiveGroups); - Literal totalSizeLiteral = new Literal(totalSize, PRIMITIVE_INT_TYPE); - expressions.add(new Invoke(buffer, "grow", totalSizeLiteral)); - Expression writerIndex = new Invoke(buffer, "writerIndex", "writerIndex", PRIMITIVE_INT_TYPE); - expressions.add(writerIndex); - int acc = 0; - for (List group : primitiveGroups) { - ListExpression groupExpressions = new ListExpression(); - for (Descriptor descriptor : group) { - int dispatchId = getNumericDescriptorDispatchId(descriptor); - Expression fieldValue = getFieldValue(bean, descriptor); - if (fieldValue instanceof Inlineable) { - ((Inlineable) fieldValue).inline(); - } - if (dispatchId == DispatchId.BOOL) { - groupExpressions.add( - bufferPutBoolean(buffer, getBufferIndex(writerIndex, acc), fieldValue)); - acc += 1; - } else if (dispatchId == DispatchId.INT8) { - groupExpressions.add(bufferPutByte(buffer, getBufferIndex(writerIndex, acc), fieldValue)); - acc += 1; - } else if (dispatchId == DispatchId.UINT8) { - groupExpressions.add( - bufferPutByte( - buffer, - getBufferIndex(writerIndex, acc), - primitiveByteValue(fieldValue, descriptor))); - acc += 1; - } else if (dispatchId == DispatchId.CHAR) { - groupExpressions.add(bufferPutChar(buffer, getBufferIndex(writerIndex, acc), fieldValue)); - acc += 2; - } else if (dispatchId == DispatchId.INT16) { - groupExpressions.add( - bufferPutInt16(buffer, getBufferIndex(writerIndex, acc), fieldValue)); - acc += 2; - } else if (dispatchId == DispatchId.UINT16) { - groupExpressions.add( - bufferPutInt16( - buffer, - getBufferIndex(writerIndex, acc), - primitiveShortValue(fieldValue, descriptor))); - acc += 2; - } else if (dispatchId == DispatchId.FLOAT16 || dispatchId == DispatchId.BFLOAT16) { - groupExpressions.add( - bufferPutInt16( - buffer, - getBufferIndex(writerIndex, acc), - new Invoke(fieldValue, "toBits", SHORT_TYPE))); - acc += 2; - } else if (dispatchId == DispatchId.INT32) { - groupExpressions.add( - bufferPutInt32(buffer, getBufferIndex(writerIndex, acc), fieldValue)); - acc += 4; - } else if (dispatchId == DispatchId.UINT32) { - groupExpressions.add( - bufferPutInt32( - buffer, - getBufferIndex(writerIndex, acc), - primitiveIntValue(fieldValue, descriptor))); - acc += 4; - } else if (dispatchId == DispatchId.INT64 || dispatchId == DispatchId.UINT64) { - groupExpressions.add( - bufferPutInt64(buffer, getBufferIndex(writerIndex, acc), fieldValue)); - acc += 8; - } else if (dispatchId == DispatchId.FLOAT32) { - groupExpressions.add( - bufferPutFloat32(buffer, getBufferIndex(writerIndex, acc), fieldValue)); - acc += 4; - } else if (dispatchId == DispatchId.FLOAT64) { - groupExpressions.add( - bufferPutFloat64(buffer, getBufferIndex(writerIndex, acc), fieldValue)); - acc += 8; - } else { - throw new IllegalStateException("Unsupported dispatchId: " + dispatchId); - } - } - if (hasFewFields() || numPrimitiveFields < 4) { - expressions.add(groupExpressions); - } else { + private void writeCompressed( + ListExpression expressions, + Expression buffer, + PrimitiveWriteAccess access, + Descriptor descriptor, + int dispatchId, + Expression fieldValue, + PrimitiveWriteState state) { + switch (dispatchId) { + case DispatchId.VARINT32: + startCompressed(expressions, buffer, state); + expressions.add(new Invoke(buffer, "_unsafeWriteVarInt32", fieldValue)); + return; + case DispatchId.VAR_UINT32: + startCompressed(expressions, buffer, state); expressions.add( - objectCodecOptimizer.invokeGenerated( - ofHashSet(bean, buffer, writerIndex), groupExpressions, "writeFields")); - } + new Invoke(buffer, "_unsafeWriteVarUInt32", primitiveIntValue(fieldValue, descriptor))); + return; + case DispatchId.VARINT64: + startCompressed(expressions, buffer, state); + expressions.add(new Invoke(buffer, "writeVarInt64", fieldValue)); + return; + case DispatchId.TAGGED_INT64: + startCompressed(expressions, buffer, state); + expressions.add(new Invoke(buffer, "writeTaggedInt64", fieldValue)); + return; + case DispatchId.VAR_UINT64: + startCompressed(expressions, buffer, state); + expressions.add(new Invoke(buffer, "writeVarUInt64", fieldValue)); + return; + case DispatchId.TAGGED_UINT64: + startCompressed(expressions, buffer, state); + expressions.add(new Invoke(buffer, "writeTaggedUInt64", fieldValue)); + return; + default: + writeFixed(expressions, access, descriptor, dispatchId, fieldValue, state); } - Expression increaseWriterIndex = - new Invoke( - buffer, - "_increaseWriterIndexUnsafe", - new Literal(totalSizeLiteral, PRIMITIVE_INT_TYPE)); - expressions.add(increaseWriterIndex); - return expressions; } - private List serializeCompressedIndexed( - Expression bean, Expression buffer, List> primitiveGroups, int totalSize) { - List expressions = new ArrayList<>(); + private void startCompressed( + ListExpression expressions, Expression buffer, PrimitiveWriteState state) { + if (!state.compressStarted) { + addIncWriterIndexExpr(expressions, buffer, state.acc); + state.compressStarted = true; + } + } + + private void writeFixed( + ListExpression expressions, + PrimitiveWriteAccess access, + Descriptor descriptor, + int dispatchId, + Expression fieldValue, + PrimitiveWriteState state) { + switch (dispatchId) { + case DispatchId.BOOL: + expressions.add(access.putBoolean(state.acc, fieldValue)); + state.acc += 1; + return; + case DispatchId.INT8: + expressions.add(access.putByte(state.acc, fieldValue)); + state.acc += 1; + return; + case DispatchId.UINT8: + expressions.add(access.putByte(state.acc, primitiveByteValue(fieldValue, descriptor))); + state.acc += 1; + return; + case DispatchId.CHAR: + expressions.add(access.putChar(state.acc, fieldValue)); + state.acc += 2; + return; + case DispatchId.INT16: + expressions.add(access.putInt16(state.acc, fieldValue)); + state.acc += 2; + return; + case DispatchId.UINT16: + expressions.add(access.putInt16(state.acc, primitiveShortValue(fieldValue, descriptor))); + state.acc += 2; + return; + case DispatchId.FLOAT16: + case DispatchId.BFLOAT16: + expressions.add(access.putInt16(state.acc, new Invoke(fieldValue, "toBits", SHORT_TYPE))); + state.acc += 2; + return; + case DispatchId.INT32: + expressions.add(access.putInt32(state.acc, fieldValue)); + state.acc += 4; + return; + case DispatchId.UINT32: + expressions.add(access.putInt32(state.acc, primitiveIntValue(fieldValue, descriptor))); + state.acc += 4; + return; + case DispatchId.INT64: + case DispatchId.UINT64: + expressions.add(access.putInt64(state.acc, fieldValue)); + state.acc += 8; + return; + case DispatchId.FLOAT32: + expressions.add(access.putFloat32(state.acc, fieldValue)); + state.acc += 4; + return; + case DispatchId.FLOAT64: + expressions.add(access.putFloat64(state.acc, fieldValue)); + state.acc += 8; + return; + default: + throw new IllegalStateException("Unsupported dispatchId: " + dispatchId); + } + } + + private int extraPrimitiveSize(List> primitiveGroups) { int extraSize = 0; for (List group : primitiveGroups) { for (Descriptor d : group) { @@ -608,6 +481,7 @@ private List serializeCompressedIndexed( || id == DispatchId.VARINT32 || id == DispatchId.VAR_UINT32 || id == DispatchId.UINT32) { + // varint may be written as 5 bytes; reserve 4 extra bytes over the fixed size. extraSize += 4; } else if (id == DispatchId.INT64 || id == DispatchId.VARINT64 @@ -615,136 +489,182 @@ private List serializeCompressedIndexed( || id == DispatchId.VAR_UINT64 || id == DispatchId.TAGGED_UINT64 || id == DispatchId.UINT64) { + // long uses 1~9 bytes; reserve one byte over the fixed size. extraSize += 1; } } } - int growSize = totalSize + extraSize; - expressions.add(new Invoke(buffer, "grow", Literal.ofInt(growSize))); - int numPrimitiveFields = getNumPrimitiveFields(primitiveGroups); - for (List group : primitiveGroups) { - ListExpression groupExpressions = new ListExpression(); - Expression writerIndex = new Invoke(buffer, "writerIndex", "writerIndex", PRIMITIVE_INT_TYPE); - int acc = 0; - boolean compressStarted = false; - for (Descriptor descriptor : group) { - int dispatchId = getNumericDescriptorDispatchId(descriptor); - Expression fieldValue = getFieldValue(bean, descriptor); - if (fieldValue instanceof Inlineable) { - ((Inlineable) fieldValue).inline(); - } - if (dispatchId == DispatchId.BOOL) { - groupExpressions.add( - bufferPutBoolean(buffer, getBufferIndex(writerIndex, acc), fieldValue)); - acc += 1; - } else if (dispatchId == DispatchId.INT8) { - groupExpressions.add(bufferPutByte(buffer, getBufferIndex(writerIndex, acc), fieldValue)); - acc += 1; - } else if (dispatchId == DispatchId.UINT8) { - groupExpressions.add( - bufferPutByte( - buffer, - getBufferIndex(writerIndex, acc), - primitiveByteValue(fieldValue, descriptor))); - acc += 1; - } else if (dispatchId == DispatchId.CHAR) { - groupExpressions.add(bufferPutChar(buffer, getBufferIndex(writerIndex, acc), fieldValue)); - acc += 2; - } else if (dispatchId == DispatchId.INT16) { - groupExpressions.add( - bufferPutInt16(buffer, getBufferIndex(writerIndex, acc), fieldValue)); - acc += 2; - } else if (dispatchId == DispatchId.UINT16) { - groupExpressions.add( - bufferPutInt16( - buffer, - getBufferIndex(writerIndex, acc), - primitiveShortValue(fieldValue, descriptor))); - acc += 2; - } else if (dispatchId == DispatchId.FLOAT16 || dispatchId == DispatchId.BFLOAT16) { - groupExpressions.add( - bufferPutInt16( - buffer, - getBufferIndex(writerIndex, acc), - new Invoke(fieldValue, "toBits", SHORT_TYPE))); - acc += 2; - } else if (dispatchId == DispatchId.FLOAT32) { - groupExpressions.add( - bufferPutFloat32(buffer, getBufferIndex(writerIndex, acc), fieldValue)); - acc += 4; - } else if (dispatchId == DispatchId.FLOAT64) { - groupExpressions.add( - bufferPutFloat64(buffer, getBufferIndex(writerIndex, acc), fieldValue)); - acc += 8; - } else if (dispatchId == DispatchId.INT32) { - groupExpressions.add( - bufferPutInt32(buffer, getBufferIndex(writerIndex, acc), fieldValue)); - acc += 4; - } else if (dispatchId == DispatchId.UINT32) { - groupExpressions.add( - bufferPutInt32( - buffer, - getBufferIndex(writerIndex, acc), - primitiveIntValue(fieldValue, descriptor))); - acc += 4; - } else if (dispatchId == DispatchId.INT64 || dispatchId == DispatchId.UINT64) { - groupExpressions.add( - bufferPutInt64(buffer, getBufferIndex(writerIndex, acc), fieldValue)); - acc += 8; - } else if (dispatchId == DispatchId.VARINT32) { - if (!compressStarted) { - addIncWriterIndexExpr(groupExpressions, buffer, acc); - compressStarted = true; - } - groupExpressions.add(new Invoke(buffer, "_unsafeWriteVarInt32", fieldValue)); - } else if (dispatchId == DispatchId.VAR_UINT32) { - if (!compressStarted) { - addIncWriterIndexExpr(groupExpressions, buffer, acc); - compressStarted = true; - } - groupExpressions.add( - new Invoke( - buffer, "_unsafeWriteVarUInt32", primitiveIntValue(fieldValue, descriptor))); - } else if (dispatchId == DispatchId.VARINT64) { - if (!compressStarted) { - addIncWriterIndexExpr(groupExpressions, buffer, acc); - compressStarted = true; - } - groupExpressions.add(new Invoke(buffer, "writeVarInt64", fieldValue)); - } else if (dispatchId == DispatchId.TAGGED_INT64) { - if (!compressStarted) { - addIncWriterIndexExpr(groupExpressions, buffer, acc); - compressStarted = true; - } - groupExpressions.add(new Invoke(buffer, "writeTaggedInt64", fieldValue)); - } else if (dispatchId == DispatchId.VAR_UINT64) { - if (!compressStarted) { - addIncWriterIndexExpr(groupExpressions, buffer, acc); - compressStarted = true; - } - groupExpressions.add(new Invoke(buffer, "writeVarUInt64", fieldValue)); - } else if (dispatchId == DispatchId.TAGGED_UINT64) { - if (!compressStarted) { - addIncWriterIndexExpr(groupExpressions, buffer, acc); - compressStarted = true; - } - groupExpressions.add(new Invoke(buffer, "writeTaggedUInt64", fieldValue)); - } else { - throw new IllegalStateException("Unsupported dispatchId: " + dispatchId); - } - } - if (!compressStarted) { - addIncWriterIndexExpr(groupExpressions, buffer, acc); - } - if (hasFewFields() || numPrimitiveFields < 4) { - expressions.add(groupExpressions); - } else { - expressions.add( - objectCodecOptimizer.invokeGenerated( - ofHashSet(bean, buffer, writerIndex), groupExpressions, "writeFields")); - } + return extraSize; + } + + private boolean useIndexedAccess() { + return JdkVersion.MAJOR_VERSION >= 25; + } + + private interface WriteAccessFactory { + PrimitiveWriteAccess get(); + } + + private static final class PrimitiveWriteState { + private int acc; + private boolean compressStarted; + + private PrimitiveWriteState(int acc) { + this.acc = acc; + } + } + + private abstract class PrimitiveWriteAccess { + protected final Expression buffer; + protected final Expression cursor; + + private PrimitiveWriteAccess(Expression buffer, Expression cursor) { + this.buffer = buffer; + this.cursor = cursor; + } + + abstract Expression putByte(int acc, Expression value); + + abstract Expression putBoolean(int acc, Expression value); + + abstract Expression putChar(int acc, Expression value); + + abstract Expression putInt16(int acc, Expression value); + + abstract Expression putInt32(int acc, Expression value); + + abstract Expression putInt64(int acc, Expression value); + + abstract Expression putFloat32(int acc, Expression value); + + abstract Expression putFloat64(int acc, Expression value); + + abstract Set fixedScope(Expression bean); + + abstract Set compressedScope(Expression bean); + } + + private final class UnsafeWriteAccess extends PrimitiveWriteAccess { + private final Expression base; + + private UnsafeWriteAccess(Expression buffer, Expression base, Expression writerAddr) { + super(buffer, writerAddr); + this.base = base; + } + + private Expression pos(int acc) { + return getWriterPos(cursor, acc); + } + + @Override + Expression putByte(int acc, Expression value) { + return unsafePut(base, pos(acc), value); + } + + @Override + Expression putBoolean(int acc, Expression value) { + return unsafePutBoolean(base, pos(acc), value); + } + + @Override + Expression putChar(int acc, Expression value) { + return unsafePutChar(base, pos(acc), value); + } + + @Override + Expression putInt16(int acc, Expression value) { + return unsafePutShort(base, pos(acc), value); + } + + @Override + Expression putInt32(int acc, Expression value) { + return unsafePutInt(base, pos(acc), value); + } + + @Override + Expression putInt64(int acc, Expression value) { + return unsafePutLong(base, pos(acc), value); + } + + @Override + Expression putFloat32(int acc, Expression value) { + return unsafePutFloat(base, pos(acc), value); + } + + @Override + Expression putFloat64(int acc, Expression value) { + return unsafePutDouble(base, pos(acc), value); + } + + @Override + Set fixedScope(Expression bean) { + return ofHashSet(bean, base, cursor); + } + + @Override + Set compressedScope(Expression bean) { + return ofHashSet(bean, buffer, base); + } + } + + private final class BufferWriteAccess extends PrimitiveWriteAccess { + private BufferWriteAccess(Expression buffer, Expression writerIndex) { + super(buffer, writerIndex); + } + + private Expression index(int acc) { + return getBufferIndex(cursor, acc); + } + + @Override + Expression putByte(int acc, Expression value) { + return bufferPutByte(buffer, index(acc), value); + } + + @Override + Expression putBoolean(int acc, Expression value) { + return bufferPutBoolean(buffer, index(acc), value); + } + + @Override + Expression putChar(int acc, Expression value) { + return bufferPutChar(buffer, index(acc), value); + } + + @Override + Expression putInt16(int acc, Expression value) { + return bufferPutInt16(buffer, index(acc), value); + } + + @Override + Expression putInt32(int acc, Expression value) { + return bufferPutInt32(buffer, index(acc), value); + } + + @Override + Expression putInt64(int acc, Expression value) { + return bufferPutInt64(buffer, index(acc), value); + } + + @Override + Expression putFloat32(int acc, Expression value) { + return bufferPutFloat32(buffer, index(acc), value); + } + + @Override + Expression putFloat64(int acc, Expression value) { + return bufferPutFloat64(buffer, index(acc), value); + } + + @Override + Set fixedScope(Expression bean) { + return ofHashSet(bean, buffer, cursor); + } + + @Override + Set compressedScope(Expression bean) { + return ofHashSet(bean, buffer, cursor); } - return expressions; } private Expression bufferPutByte(Expression buffer, Expression index, Expression value) { @@ -1066,502 +986,377 @@ protected List deserializePrimitives( private List deserializeUnCompressedPrimitives( Expression bean, Expression buffer, List> primitiveGroups, int totalSize) { - if (JdkVersion.MAJOR_VERSION >= 25) { - return deserializeRawIndexed(bean, buffer, primitiveGroups, totalSize); - } List expressions = new ArrayList<>(); - int numPrimitiveFields = getNumPrimitiveFields(primitiveGroups); Literal totalSizeLiteral = Literal.ofInt(totalSize); - // After this check, following read can be totally unsafe without checks + // After this check, following reads can use unchecked low-level access. expressions.add(new Invoke(buffer, "checkReadableBytes", totalSizeLiteral)); - Expression heapBuffer = - new Invoke(buffer, "getHeapMemory", "heapBuffer", PRIMITIVE_BYTE_ARRAY_TYPE); - Expression readerAddr = - new Invoke(buffer, "getUnsafeReaderAddress", "readerAddr", PRIMITIVE_LONG_TYPE); - expressions.add(heapBuffer); - expressions.add(readerAddr); - int acc = 0; - for (List group : primitiveGroups) { - ListExpression groupExpressions = new ListExpression(); - for (Descriptor descriptor : group) { - int dispatchId = getNumericDescriptorDispatchId(descriptor); - Expression fieldValue; - if (dispatchId == DispatchId.BOOL) { - fieldValue = unsafeGetBoolean(heapBuffer, getReaderAddress(readerAddr, acc)); - acc += 1; - } else if (dispatchId == DispatchId.INT8) { - fieldValue = unsafeGet(heapBuffer, getReaderAddress(readerAddr, acc)); - acc += 1; - } else if (dispatchId == DispatchId.UINT8) { - fieldValue = - new StaticInvoke( - Byte.class, - "toUnsignedInt", - descriptor.getTypeRef(), - unsafeGet(heapBuffer, getReaderAddress(readerAddr, acc))); - acc += 1; - } else if (dispatchId == DispatchId.CHAR) { - fieldValue = unsafeGetChar(heapBuffer, getReaderAddress(readerAddr, acc)); - acc += 2; - } else if (dispatchId == DispatchId.INT16) { - fieldValue = unsafeGetShort(heapBuffer, getReaderAddress(readerAddr, acc)); - acc += 2; - } else if (dispatchId == DispatchId.UINT16) { - fieldValue = - new StaticInvoke( - Short.class, - "toUnsignedInt", - descriptor.getTypeRef(), - unsafeGetShort(heapBuffer, getReaderAddress(readerAddr, acc))); - acc += 2; - } else if (dispatchId == DispatchId.FLOAT16) { - fieldValue = - new StaticInvoke( - Float16.class, - "fromBits", - TypeRef.of(Float16.class), - unsafeGetShort(heapBuffer, getReaderAddress(readerAddr, acc))); - acc += 2; - } else if (dispatchId == DispatchId.BFLOAT16) { - fieldValue = - new StaticInvoke( - BFloat16.class, - "fromBits", - TypeRef.of(BFloat16.class), - unsafeGetShort(heapBuffer, getReaderAddress(readerAddr, acc))); - acc += 2; - } else if (dispatchId == DispatchId.INT32) { - fieldValue = unsafeGetInt(heapBuffer, getReaderAddress(readerAddr, acc)); - acc += 4; - } else if (dispatchId == DispatchId.UINT32) { - fieldValue = - new StaticInvoke( - Integer.class, - "toUnsignedLong", - descriptor.getTypeRef(), - unsafeGetInt(heapBuffer, getReaderAddress(readerAddr, acc))); - acc += 4; - } else if (dispatchId == DispatchId.INT64 || dispatchId == DispatchId.UINT64) { - fieldValue = unsafeGetLong(heapBuffer, getReaderAddress(readerAddr, acc)); - acc += 8; - } else if (dispatchId == DispatchId.FLOAT32) { - fieldValue = unsafeGetFloat(heapBuffer, getReaderAddress(readerAddr, acc)); - acc += 4; - } else if (dispatchId == DispatchId.FLOAT64) { - fieldValue = unsafeGetDouble(heapBuffer, getReaderAddress(readerAddr, acc)); - acc += 8; - } else { - throw new IllegalStateException("Unsupported dispatchId: " + dispatchId); - } - // `bean` will be replaced by `Reference` to cut-off expr dependency. - groupExpressions.add(setFieldValue(bean, descriptor, fieldValue)); - } - if (hasFewFields() || numPrimitiveFields < 4 || isRecord) { - expressions.add(groupExpressions); - } else { - expressions.add( - objectCodecOptimizer.invokeGenerated( - ofHashSet(bean, heapBuffer, readerAddr), groupExpressions, "readFields")); - } + PrimitiveReadAccess access; + if (useIndexedAccess()) { + Expression readerIndex = new Invoke(buffer, "readerIndex", "readerIndex", PRIMITIVE_INT_TYPE); + expressions.add(readerIndex); + access = new BufferReadAccess(buffer, readerIndex); + } else { + Expression heapBuffer = + new Invoke(buffer, "getHeapMemory", "heapBuffer", PRIMITIVE_BYTE_ARRAY_TYPE); + Expression readerAddr = + new Invoke(buffer, "getUnsafeReaderAddress", "readerAddr", PRIMITIVE_LONG_TYPE); + expressions.add(heapBuffer); + expressions.add(readerAddr); + access = new UnsafeReadAccess(buffer, heapBuffer, readerAddr); } - Expression increaseReaderIndex = - new Invoke( - buffer, "increaseReaderIndex", new Literal(totalSizeLiteral, PRIMITIVE_INT_TYPE)); - expressions.add(increaseReaderIndex); + readPrimitiveGroups(expressions, bean, buffer, primitiveGroups, ignored -> access, false); + expressions.add(new Invoke(buffer, "increaseReaderIndex", totalSizeLiteral)); return expressions; } private List deserializeCompressedPrimitives( Expression bean, Expression buffer, List> primitiveGroups) { - if (JdkVersion.MAJOR_VERSION >= 25) { - return deserializeCompressedIndexed(bean, buffer, primitiveGroups); - } List expressions = new ArrayList<>(); + if (useIndexedAccess()) { + readPrimitiveGroups( + expressions, + bean, + buffer, + primitiveGroups, + readExpressions -> { + Expression readerIndex = + new Invoke(buffer, "readerIndex", "readerIndex", PRIMITIVE_INT_TYPE); + readExpressions.add(readerIndex); + return new BufferReadAccess(buffer, readerIndex); + }, + true); + } else { + readPrimitiveGroups( + expressions, + bean, + buffer, + primitiveGroups, + readExpressions -> { + // checkReadableBytes first, `fillBuffer` may create a new heap buffer. + Expression heapBuffer = + new Invoke(buffer, "getHeapMemory", "heapBuffer", PRIMITIVE_BYTE_ARRAY_TYPE); + readExpressions.add(heapBuffer); + Expression readerAddr = + new Invoke(buffer, "getUnsafeReaderAddress", "readerAddr", PRIMITIVE_LONG_TYPE); + return new UnsafeReadAccess(buffer, heapBuffer, readerAddr); + }, + true); + } + return expressions; + } + + private void readPrimitiveGroups( + List expressions, + Expression bean, + Expression buffer, + List> primitiveGroups, + ReadAccessFactory accessFactory, + boolean compressed) { int numPrimitiveFields = getNumPrimitiveFields(primitiveGroups); + int rawAcc = 0; for (List group : primitiveGroups) { - // After this check, following read can be totally unsafe without checks. - // checkReadableBytes first, `fillBuffer` may create a new heap buffer. - ReplaceStub checkReadableBytesStub = new ReplaceStub(); - expressions.add(checkReadableBytesStub); - Expression heapBuffer = - new Invoke(buffer, "getHeapMemory", "heapBuffer", PRIMITIVE_BYTE_ARRAY_TYPE); - expressions.add(heapBuffer); + ReplaceStub checkReadableBytesStub = null; + if (compressed) { + // After this check, following reads can use unchecked low-level access. + checkReadableBytesStub = new ReplaceStub(); + expressions.add(checkReadableBytesStub); + } + PrimitiveReadAccess access = accessFactory.get(expressions); ListExpression groupExpressions = new ListExpression(); - Expression readerAddr = - new Invoke(buffer, "getUnsafeReaderAddress", "readerAddr", PRIMITIVE_LONG_TYPE); - int acc = 0; - boolean compressStarted = false; + PrimitiveReadState state = new PrimitiveReadState(compressed ? 0 : rawAcc); for (Descriptor descriptor : group) { int dispatchId = getNumericDescriptorDispatchId(descriptor); - Expression fieldValue; - if (dispatchId == DispatchId.BOOL) { - fieldValue = unsafeGetBoolean(heapBuffer, getReaderAddress(readerAddr, acc)); - acc += 1; - } else if (dispatchId == DispatchId.INT8) { - fieldValue = unsafeGet(heapBuffer, getReaderAddress(readerAddr, acc)); - acc += 1; - } else if (dispatchId == DispatchId.UINT8) { - fieldValue = - new StaticInvoke( - Byte.class, - "toUnsignedInt", - descriptor.getTypeRef(), - unsafeGet(heapBuffer, getReaderAddress(readerAddr, acc))); - acc += 1; - } else if (dispatchId == DispatchId.CHAR) { - fieldValue = unsafeGetChar(heapBuffer, getReaderAddress(readerAddr, acc)); - acc += 2; - } else if (dispatchId == DispatchId.INT16) { - fieldValue = unsafeGetShort(heapBuffer, getReaderAddress(readerAddr, acc)); - acc += 2; - } else if (dispatchId == DispatchId.UINT16) { - fieldValue = - new StaticInvoke( - Short.class, - "toUnsignedInt", - descriptor.getTypeRef(), - unsafeGetShort(heapBuffer, getReaderAddress(readerAddr, acc))); - acc += 2; - } else if (dispatchId == DispatchId.FLOAT16) { - fieldValue = - new StaticInvoke( - Float16.class, - "fromBits", - TypeRef.of(Float16.class), - unsafeGetShort(heapBuffer, getReaderAddress(readerAddr, acc))); - acc += 2; - } else if (dispatchId == DispatchId.BFLOAT16) { - fieldValue = - new StaticInvoke( - BFloat16.class, - "fromBits", - TypeRef.of(BFloat16.class), - unsafeGetShort(heapBuffer, getReaderAddress(readerAddr, acc))); - acc += 2; - } else if (dispatchId == DispatchId.FLOAT32) { - fieldValue = unsafeGetFloat(heapBuffer, getReaderAddress(readerAddr, acc)); - acc += 4; - } else if (dispatchId == DispatchId.FLOAT64) { - fieldValue = unsafeGetDouble(heapBuffer, getReaderAddress(readerAddr, acc)); - acc += 8; - } else if (dispatchId == DispatchId.INT32) { - fieldValue = unsafeGetInt(heapBuffer, getReaderAddress(readerAddr, acc)); - acc += 4; - } else if (dispatchId == DispatchId.UINT32) { - fieldValue = - new StaticInvoke( - Integer.class, - "toUnsignedLong", - descriptor.getTypeRef(), - unsafeGetInt(heapBuffer, getReaderAddress(readerAddr, acc))); - acc += 4; - } else if (dispatchId == DispatchId.INT64 || dispatchId == DispatchId.UINT64) { - fieldValue = unsafeGetLong(heapBuffer, getReaderAddress(readerAddr, acc)); - acc += 8; - } else if (dispatchId == DispatchId.VARINT32) { - if (!compressStarted) { - compressStarted = true; - addIncReaderIndexExpr(groupExpressions, buffer, acc); - } - fieldValue = readVarInt32(buffer); - } else if (dispatchId == DispatchId.VAR_UINT32) { - if (!compressStarted) { - compressStarted = true; - addIncReaderIndexExpr(groupExpressions, buffer, acc); - } - fieldValue = - new StaticInvoke( - Integer.class, - "toUnsignedLong", - descriptor.getTypeRef(), - new Invoke(buffer, "readVarUInt32", PRIMITIVE_INT_TYPE)); - } else if (dispatchId == DispatchId.VARINT64) { - if (!compressStarted) { - compressStarted = true; - addIncReaderIndexExpr(groupExpressions, buffer, acc); - } - fieldValue = new Invoke(buffer, "readVarInt64", PRIMITIVE_LONG_TYPE); - } else if (dispatchId == DispatchId.TAGGED_INT64) { - if (!compressStarted) { - compressStarted = true; - addIncReaderIndexExpr(groupExpressions, buffer, acc); - } - fieldValue = new Invoke(buffer, "readTaggedInt64", PRIMITIVE_LONG_TYPE); - } else if (dispatchId == DispatchId.VAR_UINT64) { - if (!compressStarted) { - compressStarted = true; - addIncReaderIndexExpr(groupExpressions, buffer, acc); - } - fieldValue = new Invoke(buffer, "readVarUInt64", PRIMITIVE_LONG_TYPE); - } else if (dispatchId == DispatchId.TAGGED_UINT64) { - if (!compressStarted) { - compressStarted = true; - addIncReaderIndexExpr(groupExpressions, buffer, acc); - } - fieldValue = new Invoke(buffer, "readTaggedUInt64", PRIMITIVE_LONG_TYPE); - } else { - throw new IllegalStateException("Unsupported dispatchId: " + dispatchId); - } + Expression fieldValue = + compressed + ? readCompressed(groupExpressions, buffer, access, descriptor, dispatchId, state) + : readFixed(access, descriptor, dispatchId, state); // `bean` will be replaced by `Reference` to cut-off expr dependency. groupExpressions.add(setFieldValue(bean, descriptor, fieldValue)); } - if (acc != 0) { - checkReadableBytesStub.setTargetObject( - new Invoke(buffer, "checkReadableBytes", Literal.ofInt(acc))); - } - if (!compressStarted) { - addIncReaderIndexExpr(groupExpressions, buffer, acc); + if (compressed) { + if (state.acc != 0) { + checkReadableBytesStub.setTargetObject( + new Invoke(buffer, "checkReadableBytes", Literal.ofInt(state.acc))); + } + if (!state.compressStarted) { + addIncReaderIndexExpr(groupExpressions, buffer, state.acc); + } + } else { + rawAcc = state.acc; } if (hasFewFields() || numPrimitiveFields < 4 || isRecord) { expressions.add(groupExpressions); } else { expressions.add( objectCodecOptimizer.invokeGenerated( - ofHashSet(bean, buffer, heapBuffer), groupExpressions, "readFields")); + compressed ? access.compressedScope(bean) : access.fixedScope(bean), + groupExpressions, + "readFields")); } } - return expressions; } - private List deserializeRawIndexed( - Expression bean, Expression buffer, List> primitiveGroups, int totalSize) { - List expressions = new ArrayList<>(); - int numPrimitiveFields = getNumPrimitiveFields(primitiveGroups); - Literal totalSizeLiteral = Literal.ofInt(totalSize); - expressions.add(new Invoke(buffer, "checkReadableBytes", totalSizeLiteral)); - Expression readerIndex = new Invoke(buffer, "readerIndex", "readerIndex", PRIMITIVE_INT_TYPE); - expressions.add(readerIndex); - int acc = 0; - for (List group : primitiveGroups) { - ListExpression groupExpressions = new ListExpression(); - for (Descriptor descriptor : group) { - int dispatchId = getNumericDescriptorDispatchId(descriptor); - Expression fieldValue; - if (dispatchId == DispatchId.BOOL) { - fieldValue = bufferGetBoolean(buffer, getBufferIndex(readerIndex, acc)); - acc += 1; - } else if (dispatchId == DispatchId.INT8) { - fieldValue = bufferGetByte(buffer, getBufferIndex(readerIndex, acc)); - acc += 1; - } else if (dispatchId == DispatchId.UINT8) { - fieldValue = - new StaticInvoke( - Byte.class, - "toUnsignedInt", - descriptor.getTypeRef(), - bufferGetByte(buffer, getBufferIndex(readerIndex, acc))); - acc += 1; - } else if (dispatchId == DispatchId.CHAR) { - fieldValue = bufferGetChar(buffer, getBufferIndex(readerIndex, acc)); - acc += 2; - } else if (dispatchId == DispatchId.INT16) { - fieldValue = bufferGetInt16(buffer, getBufferIndex(readerIndex, acc)); - acc += 2; - } else if (dispatchId == DispatchId.UINT16) { - fieldValue = - new StaticInvoke( - Short.class, - "toUnsignedInt", - descriptor.getTypeRef(), - bufferGetInt16(buffer, getBufferIndex(readerIndex, acc))); - acc += 2; - } else if (dispatchId == DispatchId.FLOAT16) { - fieldValue = - new StaticInvoke( - Float16.class, - "fromBits", - TypeRef.of(Float16.class), - bufferGetInt16(buffer, getBufferIndex(readerIndex, acc))); - acc += 2; - } else if (dispatchId == DispatchId.BFLOAT16) { - fieldValue = - new StaticInvoke( - BFloat16.class, - "fromBits", - TypeRef.of(BFloat16.class), - bufferGetInt16(buffer, getBufferIndex(readerIndex, acc))); - acc += 2; - } else if (dispatchId == DispatchId.INT32) { - fieldValue = bufferGetInt32(buffer, getBufferIndex(readerIndex, acc)); - acc += 4; - } else if (dispatchId == DispatchId.UINT32) { - fieldValue = - new StaticInvoke( - Integer.class, - "toUnsignedLong", - descriptor.getTypeRef(), - bufferGetInt32(buffer, getBufferIndex(readerIndex, acc))); - acc += 4; - } else if (dispatchId == DispatchId.INT64 || dispatchId == DispatchId.UINT64) { - fieldValue = bufferGetInt64(buffer, getBufferIndex(readerIndex, acc)); - acc += 8; - } else if (dispatchId == DispatchId.FLOAT32) { - fieldValue = bufferGetFloat32(buffer, getBufferIndex(readerIndex, acc)); - acc += 4; - } else if (dispatchId == DispatchId.FLOAT64) { - fieldValue = bufferGetFloat64(buffer, getBufferIndex(readerIndex, acc)); - acc += 8; - } else { - throw new IllegalStateException("Unsupported dispatchId: " + dispatchId); - } - groupExpressions.add(setFieldValue(bean, descriptor, fieldValue)); - } - if (hasFewFields() || numPrimitiveFields < 4 || isRecord) { - expressions.add(groupExpressions); - } else { - expressions.add( - objectCodecOptimizer.invokeGenerated( - ofHashSet(bean, buffer, readerIndex), groupExpressions, "readFields")); - } + private Expression readCompressed( + ListExpression expressions, + Expression buffer, + PrimitiveReadAccess access, + Descriptor descriptor, + int dispatchId, + PrimitiveReadState state) { + switch (dispatchId) { + case DispatchId.VARINT32: + startReadCompressed(expressions, buffer, state); + return readVarInt32(buffer); + case DispatchId.VAR_UINT32: + startReadCompressed(expressions, buffer, state); + return new StaticInvoke( + Integer.class, + "toUnsignedLong", + descriptor.getTypeRef(), + new Invoke(buffer, "readVarUInt32", PRIMITIVE_INT_TYPE)); + case DispatchId.VARINT64: + startReadCompressed(expressions, buffer, state); + return new Invoke(buffer, "readVarInt64", PRIMITIVE_LONG_TYPE); + case DispatchId.TAGGED_INT64: + startReadCompressed(expressions, buffer, state); + return new Invoke(buffer, "readTaggedInt64", PRIMITIVE_LONG_TYPE); + case DispatchId.VAR_UINT64: + startReadCompressed(expressions, buffer, state); + return new Invoke(buffer, "readVarUInt64", PRIMITIVE_LONG_TYPE); + case DispatchId.TAGGED_UINT64: + startReadCompressed(expressions, buffer, state); + return new Invoke(buffer, "readTaggedUInt64", PRIMITIVE_LONG_TYPE); + default: + return readFixed(access, descriptor, dispatchId, state); } - Expression increaseReaderIndex = - new Invoke( - buffer, "increaseReaderIndex", new Literal(totalSizeLiteral, PRIMITIVE_INT_TYPE)); - expressions.add(increaseReaderIndex); - return expressions; } - private List deserializeCompressedIndexed( - Expression bean, Expression buffer, List> primitiveGroups) { - List expressions = new ArrayList<>(); - int numPrimitiveFields = getNumPrimitiveFields(primitiveGroups); - for (List group : primitiveGroups) { - ReplaceStub checkReadableBytesStub = new ReplaceStub(); - expressions.add(checkReadableBytesStub); - Expression readerIndex = new Invoke(buffer, "readerIndex", "readerIndex", PRIMITIVE_INT_TYPE); - expressions.add(readerIndex); - ListExpression groupExpressions = new ListExpression(); - int acc = 0; - boolean compressStarted = false; - for (Descriptor descriptor : group) { - int dispatchId = getNumericDescriptorDispatchId(descriptor); - Expression fieldValue; - if (dispatchId == DispatchId.BOOL) { - fieldValue = bufferGetBoolean(buffer, getBufferIndex(readerIndex, acc)); - acc += 1; - } else if (dispatchId == DispatchId.INT8) { - fieldValue = bufferGetByte(buffer, getBufferIndex(readerIndex, acc)); - acc += 1; - } else if (dispatchId == DispatchId.UINT8) { - fieldValue = - new StaticInvoke( - Byte.class, - "toUnsignedInt", - descriptor.getTypeRef(), - bufferGetByte(buffer, getBufferIndex(readerIndex, acc))); - acc += 1; - } else if (dispatchId == DispatchId.CHAR) { - fieldValue = bufferGetChar(buffer, getBufferIndex(readerIndex, acc)); - acc += 2; - } else if (dispatchId == DispatchId.INT16) { - fieldValue = bufferGetInt16(buffer, getBufferIndex(readerIndex, acc)); - acc += 2; - } else if (dispatchId == DispatchId.UINT16) { - fieldValue = - new StaticInvoke( - Short.class, - "toUnsignedInt", - descriptor.getTypeRef(), - bufferGetInt16(buffer, getBufferIndex(readerIndex, acc))); - acc += 2; - } else if (dispatchId == DispatchId.FLOAT16) { - fieldValue = - new StaticInvoke( - Float16.class, - "fromBits", - TypeRef.of(Float16.class), - bufferGetInt16(buffer, getBufferIndex(readerIndex, acc))); - acc += 2; - } else if (dispatchId == DispatchId.BFLOAT16) { - fieldValue = - new StaticInvoke( - BFloat16.class, - "fromBits", - TypeRef.of(BFloat16.class), - bufferGetInt16(buffer, getBufferIndex(readerIndex, acc))); - acc += 2; - } else if (dispatchId == DispatchId.FLOAT32) { - fieldValue = bufferGetFloat32(buffer, getBufferIndex(readerIndex, acc)); - acc += 4; - } else if (dispatchId == DispatchId.FLOAT64) { - fieldValue = bufferGetFloat64(buffer, getBufferIndex(readerIndex, acc)); - acc += 8; - } else if (dispatchId == DispatchId.INT32) { - fieldValue = bufferGetInt32(buffer, getBufferIndex(readerIndex, acc)); - acc += 4; - } else if (dispatchId == DispatchId.UINT32) { - fieldValue = - new StaticInvoke( - Integer.class, - "toUnsignedLong", - descriptor.getTypeRef(), - bufferGetInt32(buffer, getBufferIndex(readerIndex, acc))); - acc += 4; - } else if (dispatchId == DispatchId.INT64 || dispatchId == DispatchId.UINT64) { - fieldValue = bufferGetInt64(buffer, getBufferIndex(readerIndex, acc)); - acc += 8; - } else if (dispatchId == DispatchId.VARINT32) { - if (!compressStarted) { - compressStarted = true; - addIncReaderIndexExpr(groupExpressions, buffer, acc); - } - fieldValue = readVarInt32(buffer); - } else if (dispatchId == DispatchId.VAR_UINT32) { - if (!compressStarted) { - compressStarted = true; - addIncReaderIndexExpr(groupExpressions, buffer, acc); - } - fieldValue = - new StaticInvoke( - Integer.class, - "toUnsignedLong", - descriptor.getTypeRef(), - new Invoke(buffer, "readVarUInt32", PRIMITIVE_INT_TYPE)); - } else if (dispatchId == DispatchId.VARINT64) { - if (!compressStarted) { - compressStarted = true; - addIncReaderIndexExpr(groupExpressions, buffer, acc); - } - fieldValue = new Invoke(buffer, "readVarInt64", PRIMITIVE_LONG_TYPE); - } else if (dispatchId == DispatchId.TAGGED_INT64) { - if (!compressStarted) { - compressStarted = true; - addIncReaderIndexExpr(groupExpressions, buffer, acc); - } - fieldValue = new Invoke(buffer, "readTaggedInt64", PRIMITIVE_LONG_TYPE); - } else if (dispatchId == DispatchId.VAR_UINT64) { - if (!compressStarted) { - compressStarted = true; - addIncReaderIndexExpr(groupExpressions, buffer, acc); - } - fieldValue = new Invoke(buffer, "readVarUInt64", PRIMITIVE_LONG_TYPE); - } else if (dispatchId == DispatchId.TAGGED_UINT64) { - if (!compressStarted) { - compressStarted = true; - addIncReaderIndexExpr(groupExpressions, buffer, acc); - } - fieldValue = new Invoke(buffer, "readTaggedUInt64", PRIMITIVE_LONG_TYPE); - } else { - throw new IllegalStateException("Unsupported dispatchId: " + dispatchId); - } - groupExpressions.add(setFieldValue(bean, descriptor, fieldValue)); - } - if (acc != 0) { - checkReadableBytesStub.setTargetObject( - new Invoke(buffer, "checkReadableBytes", Literal.ofInt(acc))); - } - if (!compressStarted) { - addIncReaderIndexExpr(groupExpressions, buffer, acc); - } - if (hasFewFields() || numPrimitiveFields < 4 || isRecord) { - expressions.add(groupExpressions); - } else { - expressions.add( - objectCodecOptimizer.invokeGenerated( - ofHashSet(bean, buffer, readerIndex), groupExpressions, "readFields")); - } + private void startReadCompressed( + ListExpression expressions, Expression buffer, PrimitiveReadState state) { + if (!state.compressStarted) { + state.compressStarted = true; + addIncReaderIndexExpr(expressions, buffer, state.acc); + } + } + + private Expression readFixed( + PrimitiveReadAccess access, Descriptor descriptor, int dispatchId, PrimitiveReadState state) { + int acc = state.acc; + switch (dispatchId) { + case DispatchId.BOOL: + state.acc = acc + 1; + return access.getBoolean(acc); + case DispatchId.INT8: + state.acc = acc + 1; + return access.getByte(acc); + case DispatchId.UINT8: + state.acc = acc + 1; + return new StaticInvoke( + Byte.class, "toUnsignedInt", descriptor.getTypeRef(), access.getByte(acc)); + case DispatchId.CHAR: + state.acc = acc + 2; + return access.getChar(acc); + case DispatchId.INT16: + state.acc = acc + 2; + return access.getInt16(acc); + case DispatchId.UINT16: + state.acc = acc + 2; + return new StaticInvoke( + Short.class, "toUnsignedInt", descriptor.getTypeRef(), access.getInt16(acc)); + case DispatchId.FLOAT16: + state.acc = acc + 2; + return new StaticInvoke( + Float16.class, "fromBits", TypeRef.of(Float16.class), access.getInt16(acc)); + case DispatchId.BFLOAT16: + state.acc = acc + 2; + return new StaticInvoke( + BFloat16.class, "fromBits", TypeRef.of(BFloat16.class), access.getInt16(acc)); + case DispatchId.INT32: + state.acc = acc + 4; + return access.getInt32(acc); + case DispatchId.UINT32: + state.acc = acc + 4; + return new StaticInvoke( + Integer.class, "toUnsignedLong", descriptor.getTypeRef(), access.getInt32(acc)); + case DispatchId.INT64: + case DispatchId.UINT64: + state.acc = acc + 8; + return access.getInt64(acc); + case DispatchId.FLOAT32: + state.acc = acc + 4; + return access.getFloat32(acc); + case DispatchId.FLOAT64: + state.acc = acc + 8; + return access.getFloat64(acc); + default: + throw new IllegalStateException("Unsupported dispatchId: " + dispatchId); + } + } + + private interface ReadAccessFactory { + PrimitiveReadAccess get(List expressions); + } + + private static final class PrimitiveReadState { + private int acc; + private boolean compressStarted; + + private PrimitiveReadState(int acc) { + this.acc = acc; + } + } + + private abstract class PrimitiveReadAccess { + protected final Expression buffer; + protected final Expression cursor; + + private PrimitiveReadAccess(Expression buffer, Expression cursor) { + this.buffer = buffer; + this.cursor = cursor; + } + + abstract Expression getByte(int acc); + + abstract Expression getBoolean(int acc); + + abstract Expression getChar(int acc); + + abstract Expression getInt16(int acc); + + abstract Expression getInt32(int acc); + + abstract Expression getInt64(int acc); + + abstract Expression getFloat32(int acc); + + abstract Expression getFloat64(int acc); + + abstract Set fixedScope(Expression bean); + + abstract Set compressedScope(Expression bean); + } + + private final class UnsafeReadAccess extends PrimitiveReadAccess { + private final Expression heapBuffer; + + private UnsafeReadAccess(Expression buffer, Expression heapBuffer, Expression readerAddr) { + super(buffer, readerAddr); + this.heapBuffer = heapBuffer; + } + + private Expression pos(int acc) { + return getReaderAddress(cursor, acc); + } + + @Override + Expression getByte(int acc) { + return unsafeGet(heapBuffer, pos(acc)); + } + + @Override + Expression getBoolean(int acc) { + return unsafeGetBoolean(heapBuffer, pos(acc)); + } + + @Override + Expression getChar(int acc) { + return unsafeGetChar(heapBuffer, pos(acc)); + } + + @Override + Expression getInt16(int acc) { + return unsafeGetShort(heapBuffer, pos(acc)); + } + + @Override + Expression getInt32(int acc) { + return unsafeGetInt(heapBuffer, pos(acc)); + } + + @Override + Expression getInt64(int acc) { + return unsafeGetLong(heapBuffer, pos(acc)); + } + + @Override + Expression getFloat32(int acc) { + return unsafeGetFloat(heapBuffer, pos(acc)); + } + + @Override + Expression getFloat64(int acc) { + return unsafeGetDouble(heapBuffer, pos(acc)); + } + + @Override + Set fixedScope(Expression bean) { + return ofHashSet(bean, heapBuffer, cursor); + } + + @Override + Set compressedScope(Expression bean) { + return ofHashSet(bean, buffer, heapBuffer); + } + } + + private final class BufferReadAccess extends PrimitiveReadAccess { + private BufferReadAccess(Expression buffer, Expression readerIndex) { + super(buffer, readerIndex); + } + + private Expression index(int acc) { + return getBufferIndex(cursor, acc); + } + + @Override + Expression getByte(int acc) { + return bufferGetByte(buffer, index(acc)); + } + + @Override + Expression getBoolean(int acc) { + return bufferGetBoolean(buffer, index(acc)); + } + + @Override + Expression getChar(int acc) { + return bufferGetChar(buffer, index(acc)); + } + + @Override + Expression getInt16(int acc) { + return bufferGetInt16(buffer, index(acc)); + } + + @Override + Expression getInt32(int acc) { + return bufferGetInt32(buffer, index(acc)); + } + + @Override + Expression getInt64(int acc) { + return bufferGetInt64(buffer, index(acc)); + } + + @Override + Expression getFloat32(int acc) { + return bufferGetFloat32(buffer, index(acc)); + } + + @Override + Expression getFloat64(int acc) { + return bufferGetFloat64(buffer, index(acc)); + } + + @Override + Set fixedScope(Expression bean) { + return ofHashSet(bean, buffer, cursor); + } + + @Override + Set compressedScope(Expression bean) { + return ofHashSet(bean, buffer, cursor); } - return expressions; } private void addIncReaderIndexExpr(ListExpression expressions, Expression buffer, int diff) { diff --git a/java/fory-core/src/main/java25/org/apache/fory/memory/MemoryBuffer.java b/java/fory-core/src/main/java25/org/apache/fory/memory/MemoryBuffer.java index a89b97cb6b..fef425a5fa 100644 --- a/java/fory-core/src/main/java25/org/apache/fory/memory/MemoryBuffer.java +++ b/java/fory-core/src/main/java25/org/apache/fory/memory/MemoryBuffer.java @@ -813,6 +813,8 @@ public byte getByte(int index) { return loadByte(pos); } + // In the Java25 overlay, `_unsafe*` preserves the root MemoryBuffer unchecked-access naming. + // These methods use indexed array, ByteBuffer, and VarHandle access, not sun.misc.Unsafe. // CHECKSTYLE.OFF:MethodName public byte _unsafeGetByte(int index) { // CHECKSTYLE.ON:MethodName From 6e84c4ef8759591436d0183233ab4f65610edd8e Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Wed, 3 Jun 2026 01:26:03 +0800 Subject: [PATCH 65/69] fix(java): align jdk25 access docs and jpms tests --- .agents/languages/java.md | 12 ++++++-- README.md | 10 +++++-- docs/guide/java/index.md | 12 ++++++-- docs/guide/java/native-serialization.md | 20 ------------- docs/guide/java/troubleshooting.md | 10 +++++-- docs/guide/kotlin/index.md | 10 +++++-- docs/guide/kotlin/schema-metadata.md | 9 ++---- docs/guide/scala/index.md | 10 +++++-- .../jpms_tests/src/main/java/module-info.java | 3 -- java/README.md | 19 +++++++++++-- .../src/main/java/org/apache/fory/Fory.java | 17 +++++++++++ .../apache/fory/serializer/RegisterTest.java | 28 +++++++++++++++++++ 12 files changed, 116 insertions(+), 44 deletions(-) diff --git a/.agents/languages/java.md b/.agents/languages/java.md index aa21bfaf7a..fa8532d40b 100644 --- a/.agents/languages/java.md +++ b/.agents/languages/java.md @@ -59,10 +59,18 @@ Load this file when changing anything under `java/` or when Java drives a cross- - If changes touch GraalVM bootstrap, serializer retention, native-image metadata, or `ObjectStreamSerializer` GraalVM behavior, verify the native-image build and run the produced binary; a plain Java compile is insufficient. - Put latest-JDK or virtual-thread tests in the latest-JDK test modules with the matching compiler/profile floor, and centralize runtime-version probing in existing compatibility utilities. - For JDK25+ zero-Unsafe work, preserve serializer-family selection by type and configuration. Do not switch a type from `ObjectStreamSerializer` or another Fory serializer family to `JavaSerializer`, a JDK stream fallback, or any broad `java.* Serializable` fallback by JDK version or no-arg-constructor shape. -- JDK25+ zero-Unsafe runtime support is a JPMS named-module design. Fory core must run with `--add-opens=java.base/java.lang.invoke=org.apache.fory.core` so it can create a true trusted lookup through the private `MethodHandles.Lookup` constructor; missing this open is an invalid access configuration, not a reason to open per-package JDK internals or switch serializer/object-creation families. Do not use `ALL-UNNAMED` as the zero-Unsafe verification target. +- JDK25+ zero-Unsafe runtime support must distinguish launch shape. When Fory is on the module + path, use `--add-opens=java.base/java.lang.invoke=org.apache.fory.core`; when Fory is on the + classpath, use `--add-opens=java.base/java.lang.invoke=ALL-UNNAMED`. Missing this open is an + invalid access configuration, not a reason to open per-package JDK internals or switch + serializer/object-creation families. JPMS tests that validate named-module access should keep the + `org.apache.fory.core` target. - Do not probe JDK25+ trusted-lookup availability and turn `_JDKAccess` field-access booleans false when the required `java.base/java.lang.invoke` open is missing. Keep those access flags true on JDK25+ and let the owning trusted-lookup path raise the configuration error. - Keep JDK25+ unsafe-removal implementation invariants in agent/design docs and tests, not user guides. User guides should document user actions such as `--sun-misc-unsafe-memory-access=deny` and `java.base/java.lang.invoke` opens; do not expose internal serializer names, owner-model rationale, or avoided fallback strategies there. -- JDK25+ user docs must not require application module package opens for Fory private-field access. The only required JPMS open is `java.base/java.lang.invoke` to `org.apache.fory.core`; application module package opens are not part of this design. +- JDK25+ user docs must not require application module package opens for Fory private-field access. + The only required platform open is `java.base/java.lang.invoke`, targeted to `ALL-UNNAMED` for + classpath runs or `org.apache.fory.core` for module-path runs; application module package opens + are not part of this design. - JDK25+ final-field user docs must not tell ordinary classes to implement `java.io.Serializable`. Fory supports ordinary non-Serializable classes; mention `Serializable` only for JDK serialization hook examples or `java.*` serializability checks. diff --git a/README.md b/README.md index a05f50807b..d5031eb543 100644 --- a/README.md +++ b/README.md @@ -142,8 +142,14 @@ Gradle: implementation "org.apache.fory:fory-core:1.1.0" ``` -On JDK25+, put Fory on the module path and open `java.lang.invoke` to the Fory -core module: +On JDK25+, open `java.lang.invoke` to Fory. Use `ALL-UNNAMED` when Fory is on +the classpath: + +```bash +--add-opens=java.base/java.lang.invoke=ALL-UNNAMED +``` + +Use the Fory core module name when Fory is on the module path: ```bash --add-opens=java.base/java.lang.invoke=org.apache.fory.core diff --git a/docs/guide/java/index.md b/docs/guide/java/index.md index 5e07e611c0..37a9db48f5 100644 --- a/docs/guide/java/index.md +++ b/docs/guide/java/index.md @@ -38,7 +38,7 @@ Apache Fory™ provides blazingly fast Java object serialization with JIT compil ### Drop-in Replacement - **100% JDK Serialization Compatible**: Supports `writeObject`/`readObject`/`writeReplace`/`readResolve`/`readObjectNoData`/`Externalizable` -- **Java 8-24 Support**: Works across all modern Java versions including Java 17+ records +- **Java 8+ Support**: Works across all modern Java versions including Java 17+ records - **GraalVM Native Image**: AOT compilation support without reflection configuration - **Android API 26+ Support**: Core object serialization works on Android without runtime code generation. @@ -70,8 +70,14 @@ implementation("org.apache.fory:fory-core:1.1.0") ### JDK25+ -On JDK25+, put Fory on the module path and open `java.lang.invoke` to the Fory -core module: +On JDK25+, open `java.lang.invoke` to Fory. Use `ALL-UNNAMED` when Fory is on +the classpath: + +```bash +--add-opens=java.base/java.lang.invoke=ALL-UNNAMED +``` + +Use the Fory core module name when Fory is on the module path: ```bash --add-opens=java.base/java.lang.invoke=org.apache.fory.core diff --git a/docs/guide/java/native-serialization.md b/docs/guide/java/native-serialization.md index fb75ad2bb3..7dd6fe58dc 100644 --- a/docs/guide/java/native-serialization.md +++ b/docs/guide/java/native-serialization.md @@ -162,23 +162,6 @@ For ordinary application classes, Fory can use generated serializers and avoid J serialization-compatible path; prefer a Fory custom serializer for hot classes when the hook-based path is too expensive. -## Final Fields And Constructors - -Records are deserialized through their canonical constructor. Ordinary classes use Fory's normal -object-creation path and field setting, including final fields when the runtime supports it: - -```java -public final class User { - private final String name; - private final int age; - - public User(String name, int age) { - this.name = name; - this.age = age; - } -} -``` - ## JDK Serialization Hooks Java native mode supports the JDK serialization hooks that are part of many existing Java object @@ -189,9 +172,6 @@ models: - `readObjectNoData` - `Externalizable` -Fory native serialization remains stable across supported JDK versions when writers and readers use -the same Fory version and runtime configuration. - ```java import java.io.IOException; import java.io.ObjectInputStream; diff --git a/docs/guide/java/troubleshooting.md b/docs/guide/java/troubleshooting.md index f0286c92aa..0c0e66c8ef 100644 --- a/docs/guide/java/troubleshooting.md +++ b/docs/guide/java/troubleshooting.md @@ -150,8 +150,14 @@ fory.registerSerializer(MyClass.class, new MyClassSerializer(fory.getTypeResolve ### JDK25+ access errors -On JDK25+, if an error names `java.base/java.lang.invoke`, put Fory on the module path and open -`java.lang.invoke` to the Fory core module: +On JDK25+, if an error names `java.base/java.lang.invoke`, open `java.lang.invoke` to Fory. Use +`ALL-UNNAMED` when Fory is on the classpath: + +```bash +--add-opens=java.base/java.lang.invoke=ALL-UNNAMED +``` + +Use the Fory core module name when Fory is on the module path: ```bash --add-opens=java.base/java.lang.invoke=org.apache.fory.core diff --git a/docs/guide/kotlin/index.md b/docs/guide/kotlin/index.md index 76325897c7..391174bf70 100644 --- a/docs/guide/kotlin/index.md +++ b/docs/guide/kotlin/index.md @@ -64,8 +64,14 @@ implementation("org.apache.fory:fory-kotlin:1.1.0") ### JDK25+ -Kotlin uses the Fory Java core at runtime. On JDK25+, put Fory on the module -path and open `java.lang.invoke` to the Fory core module: +Kotlin uses the Fory Java core at runtime. On JDK25+, open `java.lang.invoke` +to Fory. Use `ALL-UNNAMED` when Fory is on the classpath: + +```bash +--add-opens=java.base/java.lang.invoke=ALL-UNNAMED +``` + +Use the Fory core module name when Fory is on the module path: ```bash --add-opens=java.base/java.lang.invoke=org.apache.fory.core diff --git a/docs/guide/kotlin/schema-metadata.md b/docs/guide/kotlin/schema-metadata.md index 7b4ed24a97..7b182ad01b 100644 --- a/docs/guide/kotlin/schema-metadata.md +++ b/docs/guide/kotlin/schema-metadata.md @@ -35,8 +35,7 @@ import org.apache.fory.kotlin.Fixed import org.apache.fory.kotlin.VarInt @ForyStruct -data class User -constructor( +data class User( @ForyField(id = 1) val id: @Fixed UInt, @@ -59,8 +58,7 @@ and maps: ```kotlin @ForyStruct -data class NullabilityExample -constructor( +data class NullabilityExample( @ForyField(id = 1) val names: List, @@ -83,8 +81,7 @@ Kotlin generated serializers preserve `@Ref` metadata for fields, list elements, import org.apache.fory.annotation.Ref @ForyStruct -data class Node -constructor( +data class Node( @ForyField(id = 1) val children: List<@Ref Node>, diff --git a/docs/guide/scala/index.md b/docs/guide/scala/index.md index c861bb0894..d60d1961a2 100644 --- a/docs/guide/scala/index.md +++ b/docs/guide/scala/index.md @@ -54,8 +54,14 @@ libraryDependencies += "org.apache.fory" %% "fory-scala" % "1.1.0" ### JDK25+ -Scala uses the Fory Java core at runtime. On JDK25+, put Fory on the module path -and open `java.lang.invoke` to the Fory core module: +Scala uses the Fory Java core at runtime. On JDK25+, open `java.lang.invoke` to +Fory. Use `ALL-UNNAMED` when Fory is on the classpath: + +```bash +--add-opens=java.base/java.lang.invoke=ALL-UNNAMED +``` + +Use the Fory core module name when Fory is on the module path: ```bash --add-opens=java.base/java.lang.invoke=org.apache.fory.core diff --git a/integration_tests/jpms_tests/src/main/java/module-info.java b/integration_tests/jpms_tests/src/main/java/module-info.java index fdda0aa8b6..be2ec6f58c 100644 --- a/integration_tests/jpms_tests/src/main/java/module-info.java +++ b/integration_tests/jpms_tests/src/main/java/module-info.java @@ -27,7 +27,4 @@ exports org.apache.fory.integration_tests.model; exports org.apache.fory.integration_tests.publicserializer; - - opens org.apache.fory.integration_tests.model to - org.apache.fory.core; } diff --git a/java/README.md b/java/README.md index 7124126cec..6b07bbea21 100644 --- a/java/README.md +++ b/java/README.md @@ -1,7 +1,7 @@ # Apache Fory™ Java [![Maven Version](https://img.shields.io/maven-central/v/org.apache.fory/fory-core?style=for-the-badge)](https://search.maven.org/#search|gav|1|g:"org.apache.fory"%20AND%20a:"fory-core") -[![Java Version](https://img.shields.io/badge/Java-8%20to%2025-blue?style=for-the-badge)](https://www.oracle.com/java/) +[![Java Version](https://img.shields.io/badge/Java-8%2B-blue?style=for-the-badge)](https://www.oracle.com/java/) [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg?style=for-the-badge)](https://opensource.org/licenses/Apache-2.0) Apache Fory™ Java provides blazingly-fast serialization for the Java ecosystem, delivering up to **170x performance improvement** over traditional frameworks through JIT compilation and zero-copy techniques. @@ -23,7 +23,7 @@ Apache Fory™ Java provides blazingly-fast serialization for the Java ecosystem ### Drop-in Replacement - **100% JDK Serialization Compatible**: Supports `writeObject`/`readObject`/`writeReplace`/`readResolve`/`readObjectNoData`/`Externalizable` -- **Java 8-24 Support**: Works across all modern Java versions including Java 17+ records +- **Java 8+ Support**: Works across all modern Java versions including Java 17+ records - **GraalVM Native Image**: AOT compilation support without reflection configuration ### Advanced Features @@ -90,6 +90,21 @@ dependencies { } ``` +### JDK25+ + +On JDK25+, open `java.lang.invoke` to Fory. Use `ALL-UNNAMED` when Fory is on +the classpath: + +```bash +--add-opens=java.base/java.lang.invoke=ALL-UNNAMED +``` + +Use the Fory core module name when Fory is on the module path: + +```bash +--add-opens=java.base/java.lang.invoke=org.apache.fory.core +``` + ## Quick Start ### Basic Usage diff --git a/java/fory-core/src/main/java/org/apache/fory/Fory.java b/java/fory-core/src/main/java/org/apache/fory/Fory.java index 39f4890c87..4a33285306 100644 --- a/java/fory-core/src/main/java/org/apache/fory/Fory.java +++ b/java/fory-core/src/main/java/org/apache/fory/Fory.java @@ -154,11 +154,13 @@ public Fory(ForyBuilder builder, ClassLoader classLoader, SharedRegistry sharedR @Override public void register(Class cls) { + checkRegisterAllowed(); getTypeResolver().register(cls); } @Override public void register(Class cls, int id) { + checkRegisterAllowed(); getTypeResolver().register(cls, Integer.toUnsignedLong(id)); } @@ -168,6 +170,7 @@ public void register(Class cls, int id) { */ @Override public void register(Class cls, String typeName) { + checkRegisterAllowed(); int idx = typeName.lastIndexOf('.'); String namespace = ""; if (idx > 0) { @@ -178,21 +181,25 @@ public void register(Class cls, String typeName) { } public void register(Class cls, String namespace, String typeName) { + checkRegisterAllowed(); getTypeResolver().register(cls, namespace, typeName); } @Override public void register(String className) { + checkRegisterAllowed(); getTypeResolver().register(className); } @Override public void register(String className, int classId) { + checkRegisterAllowed(); getTypeResolver().register(className, Integer.toUnsignedLong(classId)); } @Override public void register(String className, String namespace, String typeName) { + checkRegisterAllowed(); getTypeResolver().register(className, namespace, typeName); } @@ -202,6 +209,7 @@ public void register(ForyModule module) { if (installedModules.containsKey(module)) { return; } + checkRegisterAllowed(); installedModules.put(module, Boolean.TRUE); try { module.install(this); @@ -213,50 +221,59 @@ public void register(ForyModule module) { @Override public void registerUnion(Class cls, int id, Serializer serializer) { + checkRegisterAllowed(); getTypeResolver().registerUnion(cls, Integer.toUnsignedLong(id), serializer); } @Override public void registerUnion( Class cls, String namespace, String typeName, Serializer serializer) { + checkRegisterAllowed(); getTypeResolver().registerUnion(cls, namespace, typeName, serializer); } @Override public void registerSerializer(Class type, Class serializerClass) { + checkRegisterAllowed(); getTypeResolver().registerSerializer(type, serializerClass); } @Override public void registerSerializer(Class type, Serializer serializer) { + checkRegisterAllowed(); getTypeResolver().registerSerializer(type, serializer); } @Override public void registerSerializer( Class type, Function> serializerCreator) { + checkRegisterAllowed(); getTypeResolver().registerSerializer(type, serializerCreator.apply(typeResolver)); } @Override public void registerSerializerAndType( Class type, Class serializerClass) { + checkRegisterAllowed(); getTypeResolver().registerSerializerAndType(type, serializerClass); } @Override public void registerSerializerAndType(Class type, Serializer serializer) { + checkRegisterAllowed(); getTypeResolver().registerSerializerAndType(type, serializer); } @Override public void registerSerializerAndType( Class type, Function> serializerCreator) { + checkRegisterAllowed(); getTypeResolver().registerSerializerAndType(type, serializerCreator.apply(typeResolver)); } @Override public void registerSerializerFactory(SerializerFactory serializerFactory) { + checkRegisterAllowed(); typeResolver.registerSerializerFactory(serializerFactory); } diff --git a/java/fory-core/src/test/java/org/apache/fory/serializer/RegisterTest.java b/java/fory-core/src/test/java/org/apache/fory/serializer/RegisterTest.java index e78422a128..0b58813fea 100644 --- a/java/fory-core/src/test/java/org/apache/fory/serializer/RegisterTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/serializer/RegisterTest.java @@ -19,11 +19,14 @@ package org.apache.fory.serializer; +import java.util.concurrent.atomic.AtomicBoolean; import org.apache.fory.Fory; +import org.apache.fory.ForyModule; import org.apache.fory.ForyTestBase; import org.apache.fory.config.ForyBuilder; import org.apache.fory.context.ReadContext; import org.apache.fory.context.WriteContext; +import org.apache.fory.exception.ForyException; import org.apache.fory.resolver.TypeResolver; import org.testng.Assert; import org.testng.annotations.Test; @@ -163,6 +166,31 @@ public static class MyExt { public String id; } + @Test + public void testFrozenFacadeRegistration() { + Fory fory = + Fory.builder().withXlang(false).withCodegen(false).requireClassRegistration(false).build(); + fory.serialize(new MyExt()); + + AtomicBoolean moduleInstalled = new AtomicBoolean(); + Assert.assertThrows( + ForyException.class, + () -> fory.register((ForyModule) runtime -> moduleInstalled.set(true))); + Assert.assertFalse(moduleInstalled.get()); + + AtomicBoolean creatorCalled = new AtomicBoolean(); + Assert.assertThrows( + ForyException.class, + () -> + fory.registerSerializer( + MyExt.class, + resolver -> { + creatorCalled.set(true); + return new MyExtSerializer(resolver); + })); + Assert.assertFalse(creatorCalled.get()); + } + public static class MyExtSerializer extends Serializer { public MyExtSerializer(TypeResolver typeResolver) { super(typeResolver.getConfig(), MyExt.class); From 199a47dce64ce6d3dcf252b9e15f933f3f17a873 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Wed, 3 Jun 2026 05:45:17 +0800 Subject: [PATCH 66/69] fix(java): support graalvm jdk25 object stream instantiation --- .agents/languages/java.md | 64 ++++-- .../fory/graalvm/FeatureTestExample.java | 6 +- java/fory-core/pom.xml | 213 +++++++++++------- .../fory/platform/internal/_JDKAccess.java | 25 ++ .../fory/reflect/ObjectInstantiators.java | 71 +++++- .../fory/serializer/CodegenSerializer.java | 8 + .../fory-core/native-image.properties | 4 + .../graalvm/feature/ForyGraalVMFeature.java | 15 ++ 8 files changed, 311 insertions(+), 95 deletions(-) diff --git a/.agents/languages/java.md b/.agents/languages/java.md index fa8532d40b..041baa0513 100644 --- a/.agents/languages/java.md +++ b/.agents/languages/java.md @@ -81,17 +81,24 @@ Load this file when changing anything under `java/` or when Java drives a cross- user explicitly asks for one. Keep the `java.base/java.lang.invoke` open in install-facing docs such as the Java/Kotlin/Scala install sections and README. - JDK25+ zero-Unsafe final-field writes must use a true target-class trusted lookup from the original `IMPL_LOOKUP`, not `IMPL_LOOKUP.in(type)`. JDK26+ normal Fory final-field restoration must pass with `--illegal-final-field-mutation=deny` and must not require `--enable-final-field-mutation`. -- For JDK25+ object creation, do not use `sun.reflect.ReflectionFactory`, - `jdk.unsupported`, `ObjectStreamClass.newInstance`, or an Unsafe-backed object instantiator. - The shared `ObjectInstantiators` facade should route supported no-constructor construction - through `ParentNoArgCtrInstantiator`, whose JDK25+ path owns trusted-lookup access to - `jdk.internal.reflect.ReflectionFactory` in `java.base`. This must not require - `--add-opens=java.base/jdk.internal.reflect=...`; the only JDK25+ platform open remains +- For JDK25+ object creation, do not use `sun.reflect.ReflectionFactory`, `jdk.unsupported`, or an + Unsafe-backed object instantiator. The shared `ObjectInstantiators` facade should route normal JVM + no-constructor construction through `ParentNoArgCtrInstantiator`, whose JDK25+ path owns + trusted-lookup access to `jdk.internal.reflect.ReflectionFactory` in `java.base`. This must not + require `--add-opens=java.base/jdk.internal.reflect=...`; the only JDK25+ platform open remains `java.base/java.lang.invoke=org.apache.fory.core`. ReflectionFactory serialization constructors also support non-Serializable ordinary classes; the normal object-instantiation path must not - reject them with ObjectStream-only parent-constructor checks. Classes unsupported by - ReflectionFactory itself require an accessible no-arg constructor, a record canonical constructor - path, or a custom serializer. + reject them with ObjectStream-only parent-constructor checks. GraalVM JDK25+ native-image + Serializable empty-instance construction is the narrow exception: direct ReflectionFactory + serialization constructors can produce `Object` there, so it uses a cached `ObjectStreamClass` + and a private `ObjectStreamClass.newInstance` MethodHandle from `_JDKAccess._trustedLookup`. If + the instantiator is retained in the native-image heap, the MethodHandle owner may be initialized at + build time but the per-type `ObjectStreamClass` descriptor must be cached only at image runtime. + `ForyGraalVMFeature` must register the matching serialization constructor target class for each + Serializable hierarchy member so runtime-lazy `ObjectStreamClass.lookupAny` can build descriptors + for JDK classes such as `TreeMap` and `TreeSet`. Classes unsupported by ReflectionFactory itself + require an accessible no-arg constructor, a record canonical constructor path, or a custom + serializer. - `UnsafeObjectInstantiator` is the JDK8-24 Unsafe owner only. It must be a top-level instantiator with a Java25 multi-release stub that contains no Unsafe, ObjectStream, ReflectionFactory, or constructor-bypass implementation. @@ -185,12 +192,30 @@ Load this file when changing anything under `java/` or when Java drives a cross- private instance helpers to static bridge methods whose receiver parameter uses the original binary class name, which fails hidden-class verification. Use non-private final split helpers on that path. +- Runtime codegen must not emit Janino source that names bootstrap JDK implementation classes in + concealed or non-source-public packages. Generated source in the unnamed module cannot access + those classes even when Fory's trusted field-access path can read/write their fields; use + `ObjectSerializer` for those object-copy/field paths. - Keep JDK26 `--illegal-final-field-mutation=deny` scoped to the JPMS runtime tests that prove Fory's field-access path. Do not put it in global Maven `JDK_JAVA_OPTIONS`, because build tools such as Lombok may perform their own reflective final-field access during compilation. -- JDK25+ collection serializers must fail unsupported `Collections.newSetFromMap` backing maps - before writing or copying. Do not rewrite them to `HashMap`, because that changes equality - semantics and can drop entries. +- JDK25+ collection serializers can restore JDK private/final collection fields through the + required trusted lookup. Do not add JDK25-only unsupported branches for `Collections.newSetFromMap` + or similar JDK wrappers when the normal JDK field-access owner can preserve the existing payload. +- Keep all private `Collections$SetFromMap` field access in one `SetFromMapAccess` owner helper. + The non-codegen payload branch is the backing-map payload path used to preserve backing-map + object/reference semantics, not a legacy fallback. +- Do not add a new built-in class to the auto-assigned `registerInternalSerializer` sequence unless + it has an explicit stable internal type id. Use `ClassResolver#getSerializerClass` for native-only + serializer selection when the class was not previously internally registered; shifting built-in + type ids breaks Java native-mode binary compatibility. +- When a native-only serializer is selected for a class that was already routed through a different + serializer family, do not shift built-in type ids. Preserve the existing payload only if it was + semantically valid; otherwise use the owning semantic serializer path and cover the behavior with + focused tests. +- When `ClassResolver#getSerializerClass` selects a serializer that is not internally registered, + add that serializer class to `GraalvmSupport`'s default serializer metadata so native-image can + construct it without shifting Java native-mode type ids. - Do not add or restore constructor-binding APIs such as `@ForyConstructor` or `BaseFory.registerConstructor(...)`. Java parameter names, `-parameters`, and `@ConstructorProperties` are not a Fory object-creation contract. Runtime serializers for @@ -199,6 +224,9 @@ Load this file when changing anything under `java/` or when Java drives a cross- - Source-generated constructor serializers must own their constructor metadata at generation time and call constructors directly. They must not depend on runtime `ObjectInstantiator` constructor-field metadata or varargs constructor calls. +- Java annotation-processor static serializers do not own ordinary-class constructor metadata. + Reject ordinary non-record final fields instead of generating descriptor-based final-field + mutation; records and Kotlin KSP primary-constructor serializers are the constructor-owned paths. - Generated JVM copy code may direct-copy immutable scalar values, but Java `Collection`/`Map` subclasses must be copied through `CopyContext.copyObject(...)` so collection/map serializers own concrete type, comparator, wrapper, and reference behavior. @@ -206,11 +234,19 @@ Load this file when changing anything under `java/` or when Java drives a cross- `cause` and `suppressedExceptions` sentinels directly before exposing the object. Do not call `initCause` or `addSuppressed` on a constructor-bypassed `Throwable` whose sentinels are still absent. -- For JDK25+ CI, do not run core runtime tests from raw Maven reactor `target/classes`. Those classes bypass `META-INF/versions/25` and exercise the JDK8-24 root implementation. Build/install the multi-release artifact first, then verify the zero-Unsafe path through the JPMS module-path suite where `org.apache.fory.core` is the real access target. +- For JDK25+ CI, do not run core runtime tests from raw Maven reactor `target/classes`. Those + classes bypass `META-INF/versions/25` and exercise the JDK8-24 root implementation. Classpath + Surefire tests must use a test-only classes directory overlaid with Java16 and Java25 replacement + classes and without `module-info.class`, so the run stays unnamed while exercising the + zero-Unsafe classes. JPMS tests still own named-module coverage where `org.apache.fory.core` is + the real access target. - After shading Janino into `fory-core`, refresh the JPMS module descriptor package table from the final jar before install. Otherwise named-module codegen cannot load concealed shaded Janino packages even though the classes are present in the jar. -- Do not make GraalVM native-image JDK25+ pass by opening `java.lang.invoke` to `ALL-UNNAMED`. Keep zero-Unsafe verification on JPMS JVM tests unless the native-image path itself runs Fory as a named module and the produced binary passes. +- GraalVM native-image tests that run Fory on the classpath may target + `java.base/java.lang.invoke=ALL-UNNAMED`; that is the classpath launch shape, not a named-module + proof. Keep named-module zero-Unsafe verification in JPMS tests unless a native-image path itself + runs Fory as `org.apache.fory.core`. ## Key Modules diff --git a/integration_tests/graalvm_tests/src/main/java/org/apache/fory/graalvm/FeatureTestExample.java b/integration_tests/graalvm_tests/src/main/java/org/apache/fory/graalvm/FeatureTestExample.java index 1e0f17d7bc..eef5992ecf 100644 --- a/integration_tests/graalvm_tests/src/main/java/org/apache/fory/graalvm/FeatureTestExample.java +++ b/integration_tests/graalvm_tests/src/main/java/org/apache/fory/graalvm/FeatureTestExample.java @@ -19,6 +19,7 @@ package org.apache.fory.graalvm; +import java.io.Serializable; import java.lang.reflect.InvocationHandler; import java.lang.reflect.Method; import java.lang.reflect.Proxy; @@ -47,7 +48,9 @@ public interface TestInterface { String getValue(); } - public static class TestInvocationHandler implements InvocationHandler { + public static class TestInvocationHandler implements InvocationHandler, Serializable { + private static final long serialVersionUID = 1L; + private final String value; public TestInvocationHandler(String value) { @@ -98,6 +101,7 @@ public static void main(String[] args) { // Test proxy serialization TestInterface proxy = + (TestInterface) Proxy.newProxyInstance( TestInterface.class.getClassLoader(), new Class[] {TestInterface.class}, diff --git a/java/fory-core/pom.xml b/java/fory-core/pom.xml index 2f3a26283a..00dac76a44 100644 --- a/java/fory-core/pom.xml +++ b/java/fory-core/pom.xml @@ -40,6 +40,7 @@ 8 ${basedir}/.. ${project.build.directory}/multi-release-classes + ${project.build.directory}/jdk25-test-classes @@ -148,37 +149,37 @@ org.apache.maven.plugins - maven-compiler-plugin + maven-antrun-plugin + 3.1.0 compile-java9-module-info process-classes - compile + run - 9 - none - - ${project.basedir}/src/main/java9 - - - module-info.java - - ${fory.mr.classes}/9 - - --patch-module - org.apache.fory.core=${project.build.outputDirectory} - + + + + + + + + + + + + + - - - - org.apache.maven.plugins - maven-antrun-plugin - 3.1.0 - inject-multi-release-classes package @@ -209,31 +210,40 @@ org.apache.maven.plugins - maven-compiler-plugin + maven-antrun-plugin + 3.1.0 compile-java16-sources process-classes - compile + run - 16 - 16 - none - - ${project.basedir}/src/main/java16 - - - **/*.java - - ${fory.mr.classes}/16 - - --add-modules - jdk.incubator.vector - --patch-module - org.apache.fory.core=${project.build.outputDirectory} - + + + + + + + + + + + + + + + + + + @@ -263,62 +273,100 @@ org.apache.maven.plugins - maven-compiler-plugin + maven-antrun-plugin + 3.1.0 compile-jdk25-classes process-classes - compile + run - 22 - none - - ${project.basedir}/src/main/java25 - - - org/**/*.java - - ${fory.mr.classes}/25 - - -sourcepath - - + + + + + + + + + + + + + + + compile-jdk25-module-info process-classes - compile + run - 22 - 22 - none - - ${project.basedir}/src/main/java25 - - - module-info.java - - ${fory.mr.classes}/25 - - --add-modules - jdk.incubator.vector - --patch-module - org.apache.fory.core=${project.build.outputDirectory}${path.separator}${fory.mr.classes}/25 - + + + + + + + + + + + + + + + + + + + + prepare-jdk25-test-classes + process-test-classes + + run + + + + + + + + + + + + + + + + + + + + + + - - - - org.apache.maven.plugins - maven-antrun-plugin - 3.1.0 - verify-jdk25-multi-release-jar verify @@ -400,6 +448,13 @@ + + org.apache.maven.plugins + maven-surefire-plugin + + ${fory.jdk25.test.classes} + + diff --git a/java/fory-core/src/main/java/org/apache/fory/platform/internal/_JDKAccess.java b/java/fory-core/src/main/java/org/apache/fory/platform/internal/_JDKAccess.java index 31792c06bf..4d67f0b193 100644 --- a/java/fory-core/src/main/java/org/apache/fory/platform/internal/_JDKAccess.java +++ b/java/fory-core/src/main/java/org/apache/fory/platform/internal/_JDKAccess.java @@ -296,6 +296,7 @@ public static Object makeGetterFunction( } private static volatile Method getModuleMethod; + private static volatile Method isExportedMethod; public static Object getModule(Class cls) { Preconditions.checkArgument(JdkVersion.MAJOR_VERSION >= 9); @@ -313,6 +314,30 @@ public static Object getModule(Class cls) { } } + public static boolean isExported(Class cls) { + if (JdkVersion.MAJOR_VERSION < 9) { + return true; + } + if (cls.isArray()) { + return isExported(cls.getComponentType()); + } + if (cls.isPrimitive()) { + return true; + } + try { + if (isExportedMethod == null) { + Class moduleClass = Class.forName("java.lang.Module"); + isExportedMethod = moduleClass.getDeclaredMethod("isExported", String.class); + } + Package pkg = cls.getPackage(); + return (Boolean) isExportedMethod.invoke(getModule(cls), pkg == null ? "" : pkg.getName()); + } catch (ClassNotFoundException | NoSuchMethodException e) { + throw new RuntimeException(e); + } catch (IllegalAccessException | InvocationTargetException e) { + throw ExceptionUtils.throwException(e); + } + } + // caller sensitive, must use MethodHandle to walk around the check. private static volatile MethodHandle addReadsHandle; diff --git a/java/fory-core/src/main/java/org/apache/fory/reflect/ObjectInstantiators.java b/java/fory-core/src/main/java/org/apache/fory/reflect/ObjectInstantiators.java index 672f4588ba..7f2925c5cf 100644 --- a/java/fory-core/src/main/java/org/apache/fory/reflect/ObjectInstantiators.java +++ b/java/fory-core/src/main/java/org/apache/fory/reflect/ObjectInstantiators.java @@ -19,6 +19,7 @@ package org.apache.fory.reflect; +import java.io.ObjectStreamClass; import java.io.Serializable; import java.lang.invoke.MethodHandle; import java.lang.invoke.MethodHandles; @@ -102,7 +103,14 @@ public static ObjectInstantiator createObjectInstantiator(Class type) if (noArgConstructor != null) { return new DeclaredNoArgCtrInstantiator<>(type); } else if (JdkVersion.MAJOR_VERSION >= 25) { - return new ParentNoArgCtrInstantiator<>(type); + if (Serializable.class.isAssignableFrom(type)) { + return new ObjectStreamInstantiator<>(type); + } + return new UnsupportedObjectInstantiator<>( + type, + "GraalVM native image on JDK25+ cannot create " + + type + + " without an accessible no-arg constructor or Serializable metadata"); } else { return new UnsafeObjectInstantiator<>(type); } @@ -132,6 +140,9 @@ public static ObjectInstantiator createObjectStreamInstantiator(Class if (GraalvmSupport.IN_GRAALVM_NATIVE_IMAGE && JdkVersion.MAJOR_VERSION < 25) { return new UnsafeObjectInstantiator<>(type); } + if (GraalvmSupport.IN_GRAALVM_NATIVE_IMAGE && JdkVersion.MAJOR_VERSION >= 25) { + return new ObjectStreamInstantiator<>(type); + } return new ParentNoArgCtrInstantiator<>(type); } @@ -268,6 +279,64 @@ public T newInstanceWithArguments(Object... arguments) { } } + public static final class ObjectStreamInstantiator extends ObjectInstantiator { + private volatile ObjectStreamClass objectStreamClass; + + public ObjectStreamInstantiator(Class type) { + super(type); + if (!GraalvmSupport.isGraalBuildTime()) { + objectStreamClass = ObjectStreamClass.lookupAny(type); + } + } + + @Override + public T newInstance() { + try { + return type.cast(ObjectStreamAccess.newInstance(objectStreamClass())); + } catch (Throwable e) { + throw makeException(type, e); + } + } + + @Override + public T newInstanceWithArguments(Object... arguments) { + throw new UnsupportedOperationException(); + } + + private ObjectStreamClass objectStreamClass() { + ObjectStreamClass localObjectStreamClass = objectStreamClass; + if (localObjectStreamClass == null) { + // Static Fory instances can be built into GraalVM native images. A build-time + // ObjectStreamClass can retain GraalVM's bad ReflectionFactory constructor, so cache the + // descriptor only after the image reaches runtime. + localObjectStreamClass = ObjectStreamClass.lookupAny(type); + objectStreamClass = localObjectStreamClass; + } + return localObjectStreamClass; + } + } + + private static final class ObjectStreamAccess { + private static final MethodHandle NEW_INSTANCE = newInstanceHandle(); + + private static Object newInstance(ObjectStreamClass objectStreamClass) throws Throwable { + return NEW_INSTANCE.invoke(objectStreamClass); + } + + private static MethodHandle newInstanceHandle() { + try { + // GraalVM JDK25+ native image handles ObjectStreamClass allocation through its + // serialization metadata. Direct ReflectionFactory serialization constructors can produce + // Object instances there, so Serializable empty-instance creation must use this owner path. + return _JDKAccess._trustedLookup(ObjectStreamClass.class) + .findVirtual( + ObjectStreamClass.class, "newInstance", MethodType.methodType(Object.class)); + } catch (Throwable e) { + throw new ExceptionInInitializerError(e); + } + } + } + public static final class ParentNoArgCtrInstantiator extends ObjectInstantiator { private final Constructor constructor; diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/CodegenSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/CodegenSerializer.java index 57ebcb10ad..4bc3b1a427 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/CodegenSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/CodegenSerializer.java @@ -25,15 +25,23 @@ import org.apache.fory.Fory; import org.apache.fory.builder.CodecUtils; import org.apache.fory.builder.Generated; +import org.apache.fory.codegen.CodeGenerator; import org.apache.fory.context.ReadContext; import org.apache.fory.context.WriteContext; import org.apache.fory.platform.AndroidSupport; +import org.apache.fory.platform.internal._JDKAccess; import org.apache.fory.resolver.TypeResolver; /** Util for JIT Serialization. */ public final class CodegenSerializer { public static boolean supportCodegenForJavaSerialization(Class cls) { + if (cls.getClassLoader() == null + && (!_JDKAccess.isExported(cls) || !CodeGenerator.sourcePublicAccessible(cls))) { + // Source generated into the unnamed module cannot legally name concealed or package-private + // JDK implementation classes. Field-access serializers own those cases. + return false; + } // bean class can be static nested class, but can't be a non-static inner class. // Check modifiers first to avoid loading the enclosing class unnecessarily — // in classloader-isolated environments (e.g. OSGi, module systems) the enclosing diff --git a/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties b/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties index a0d3dce15e..af974f69da 100644 --- a/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties +++ b/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties @@ -248,6 +248,7 @@ Args=--initialize-at-build-time=org.apache.fory.annotation.ForyField$Dynamic,\ org.apache.fory.meta.TypeEqualMetaCompressor,\ org.apache.fory.pool.ThreadPoolFory,\ org.apache.fory.reflect.InstanceFieldAccessors,\ + org.apache.fory.reflect.InstanceFieldAccessors$InstanceAccessor,\ org.apache.fory.reflect.InstanceFieldAccessors$GeneratedAccessor,\ org.apache.fory.reflect.ReflectionUtils$1,\ org.apache.fory.reflect.ReflectionUtils$2,\ @@ -259,8 +260,11 @@ Args=--initialize-at-build-time=org.apache.fory.annotation.ForyField$Dynamic,\ org.apache.fory.reflect.TypeRef,\ org.apache.fory.reflect.ObjectInstantiators,\ org.apache.fory.reflect.ObjectInstantiators$DeclaredNoArgCtrInstantiator,\ + org.apache.fory.reflect.ObjectInstantiators$ObjectStreamInstantiator,\ + org.apache.fory.reflect.ObjectInstantiators$ObjectStreamAccess,\ org.apache.fory.reflect.ObjectInstantiators$ParentNoArgCtrInstantiator,\ org.apache.fory.reflect.ObjectInstantiators$RecordObjectInstantiator,\ + org.apache.fory.reflect.ObjectInstantiators$UnsupportedObjectInstantiator,\ org.apache.fory.resolver.TypeChecker,\ org.apache.fory.resolver.TypeInfo,\ org.apache.fory.resolver.TypeInfoHolder,\ diff --git a/java/fory-graalvm-feature/src/main/java/org/apache/fory/graalvm/feature/ForyGraalVMFeature.java b/java/fory-graalvm-feature/src/main/java/org/apache/fory/graalvm/feature/ForyGraalVMFeature.java index 51cd87fdf8..bf55617fa7 100644 --- a/java/fory-graalvm-feature/src/main/java/org/apache/fory/graalvm/feature/ForyGraalVMFeature.java +++ b/java/fory-graalvm-feature/src/main/java/org/apache/fory/graalvm/feature/ForyGraalVMFeature.java @@ -139,6 +139,21 @@ private void registerSerializableHierarchy(Class clazz) { current != null && current != Object.class && Serializable.class.isAssignableFrom(current); current = current.getSuperclass()) { RuntimeSerialization.registerIncludingAssociatedClasses(current); + registerSerializationConstructor(current); + } + } + + private void registerSerializationConstructor(Class clazz) { + Class targetConstructorClass = clazz.getSuperclass(); + while (targetConstructorClass != null + && Serializable.class.isAssignableFrom(targetConstructorClass)) { + targetConstructorClass = targetConstructorClass.getSuperclass(); + } + if (targetConstructorClass != null) { + // JDK25+ Fory can lazily build ObjectStreamClass descriptors at image runtime. GraalVM needs + // the matching serialization constructor accessor pre-registered for that target superclass, + // or ObjectStreamClass.lookupAny can fail for JDK classes such as TreeMap and TreeSet. + RuntimeSerialization.registerWithTargetConstructorClass(clazz, targetConstructorClass); } } From d001d1f653dbd7ff57f4bdad661e7817685f6e0f Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Wed, 3 Jun 2026 07:04:10 +0800 Subject: [PATCH 67/69] fix(java): stabilize jdk25 ci paths --- .agents/languages/java.md | 35 ++--- benchmarks/java/pom.xml | 1 + compiler/fory_compiler/generators/rust.py | 16 ++- .../tests/test_generated_code.py | 31 +++++ docs/guide/java/type-registration.md | 3 + .../JpmsFieldAccessorTest.java | 31 ++++- .../processing/ForyStructProcessor.java | 18 ++- .../annotation/processing/SourceField.java | 3 - .../StaticSerializerSourceWriter.java | 129 +++--------------- .../processing/ForyStructProcessorTest.java | 48 ------- .../builder/StaticCompatibleCodecBuilder.java | 11 +- .../apache/fory/platform/GraalvmSupport.java | 1 + .../fory/platform/internal/DefineClass.java | 11 +- .../fory/platform/internal/_JDKAccess.java | 7 + .../fory/reflect/ObjectInstantiators.java | 111 ++++++++++----- .../apache/fory/resolver/ClassResolver.java | 4 + .../fory/serializer/ExceptionSerializers.java | 17 ++- .../collection/CollectionSerializers.java | 81 ++++------- .../serializer/collection/MapSerializers.java | 19 +++ .../java/org/apache/fory/type/TypeUtils.java | 5 +- .../org/apache/fory/memory/MemoryBuffer.java | 2 +- .../fory/platform/internal/_Lookup.java | 3 +- .../fory/reflect/InstanceFieldAccessors.java | 5 +- .../fory/serializer/PlatformStringUtils.java | 3 +- .../fory-core/native-image.properties | 2 + .../test/java/org/apache/fory/ForyTest.java | 27 ++++ .../fory/reflect/ObjectInstantiatorsTest.java | 19 +-- .../fory/reflect/ReflectionUtilsTest.java | 12 +- .../fory/serializer/ObjectSerializerTest.java | 53 +++++++ .../AndroidCollectionFeatureTest.java | 50 +++++-- .../collection/CollectionSerializersTest.java | 24 +++- .../collection/MapSerializersTest.java | 90 ++++++++---- .../ImmutableCollectionSerializersTest.java | 32 +++-- java/fory-testsuite/pom.xml | 4 +- 34 files changed, 527 insertions(+), 381 deletions(-) diff --git a/.agents/languages/java.md b/.agents/languages/java.md index 041baa0513..60bf283305 100644 --- a/.agents/languages/java.md +++ b/.agents/languages/java.md @@ -82,23 +82,24 @@ Load this file when changing anything under `java/` or when Java drives a cross- such as the Java/Kotlin/Scala install sections and README. - JDK25+ zero-Unsafe final-field writes must use a true target-class trusted lookup from the original `IMPL_LOOKUP`, not `IMPL_LOOKUP.in(type)`. JDK26+ normal Fory final-field restoration must pass with `--illegal-final-field-mutation=deny` and must not require `--enable-final-field-mutation`. - For JDK25+ object creation, do not use `sun.reflect.ReflectionFactory`, `jdk.unsupported`, or an - Unsafe-backed object instantiator. The shared `ObjectInstantiators` facade should route normal JVM - no-constructor construction through `ParentNoArgCtrInstantiator`, whose JDK25+ path owns - trusted-lookup access to `jdk.internal.reflect.ReflectionFactory` in `java.base`. This must not - require `--add-opens=java.base/jdk.internal.reflect=...`; the only JDK25+ platform open remains - `java.base/java.lang.invoke=org.apache.fory.core`. ReflectionFactory serialization constructors - also support non-Serializable ordinary classes; the normal object-instantiation path must not - reject them with ObjectStream-only parent-constructor checks. GraalVM JDK25+ native-image - Serializable empty-instance construction is the narrow exception: direct ReflectionFactory - serialization constructors can produce `Object` there, so it uses a cached `ObjectStreamClass` - and a private `ObjectStreamClass.newInstance` MethodHandle from `_JDKAccess._trustedLookup`. If - the instantiator is retained in the native-image heap, the MethodHandle owner may be initialized at - build time but the per-type `ObjectStreamClass` descriptor must be cached only at image runtime. - `ForyGraalVMFeature` must register the matching serialization constructor target class for each - Serializable hierarchy member so runtime-lazy `ObjectStreamClass.lookupAny` can build descriptors - for JDK classes such as `TreeMap` and `TreeSet`. Classes unsupported by ReflectionFactory itself - require an accessible no-arg constructor, a record canonical constructor path, or a custom - serializer. + Unsafe-backed object instantiator. Normal JVM no-constructor construction must use the + `ObjectInstantiators` ReflectionFactory bypass path with `Object` as the template constructor, so + normal Fory field restoration does not inherit ObjectStream-only first-non-Serializable-superclass + constructor validation. ObjectStream-compatible serializers own the separate + `ParentNoArgCtrInstantiator` path and must keep Java serialization parent-constructor rules. The + JDK25+ ReflectionFactory path uses trusted-lookup access to `jdk.internal.reflect.ReflectionFactory` + in `java.base` and must not require `--add-opens=java.base/jdk.internal.reflect=...`; the only + JDK25+ platform open remains `java.base/java.lang.invoke=org.apache.fory.core`. GraalVM JDK25+ + native-image Serializable empty-instance construction is the narrow exception: direct + ReflectionFactory serialization constructors can produce `Object` there, so it uses a cached + `ObjectStreamClass` and a private `ObjectStreamClass.newInstance` MethodHandle from + `_JDKAccess._trustedLookup`. If the instantiator is retained in the native-image heap, the + MethodHandle owner may be initialized at build time but the per-type `ObjectStreamClass` descriptor + must be cached only at image runtime. `ForyGraalVMFeature` must register the matching serialization + constructor target class for each Serializable hierarchy member so runtime-lazy + `ObjectStreamClass.lookupAny` can build descriptors for JDK classes such as `TreeMap` and + `TreeSet`. Classes unsupported by ReflectionFactory itself require an accessible no-arg + constructor, a record canonical constructor path, or a custom serializer. - `UnsafeObjectInstantiator` is the JDK8-24 Unsafe owner only. It must be a top-level instantiator with a Java25 multi-release stub that contains no Unsafe, ObjectStream, ReflectionFactory, or constructor-bypass implementation. diff --git a/benchmarks/java/pom.xml b/benchmarks/java/pom.xml index 660b99e52e..91a46a715d 100644 --- a/benchmarks/java/pom.xml +++ b/benchmarks/java/pom.xml @@ -348,6 +348,7 @@ + diff --git a/compiler/fory_compiler/generators/rust.py b/compiler/fory_compiler/generators/rust.py index 6d722bde39..652a9cd6eb 100644 --- a/compiler/fory_compiler/generators/rust.py +++ b/compiler/fory_compiler/generators/rust.py @@ -1039,12 +1039,18 @@ def generate_type( elif isinstance(field_type, ListType): effective_element_optional = element_optional or field_type.element_optional effective_element_ref = element_ref or field_type.element_ref + element_pointer_type = pointer_type + if field_type.element_ref: + element_pointer_type = self.get_pointer_type( + field_type.element_ref_options, + field_type.element_ref_options.get("weak_ref") is True, + ) element_type = self.generate_type( field_type.element_type, nullable=effective_element_optional, ref=effective_element_ref, parent_stack=parent_stack, - pointer_type=pointer_type, + pointer_type=element_pointer_type, ) list_type = f"::std::vec::Vec<{element_type}>" if ref: @@ -1076,12 +1082,18 @@ def generate_type( parent_stack=parent_stack, pointer_type=pointer_type, ) + value_pointer_type = pointer_type + if field_type.value_ref: + value_pointer_type = self.get_pointer_type( + field_type.value_ref_options, + field_type.value_ref_options.get("weak_ref") is True, + ) value_type = self.generate_type( field_type.value_type, nullable=False, ref=field_type.value_ref, parent_stack=parent_stack, - pointer_type=pointer_type, + pointer_type=value_pointer_type, ) map_type = f"::std::collections::HashMap<{key_type}, {value_type}>" if ref: diff --git a/compiler/fory_compiler/tests/test_generated_code.py b/compiler/fory_compiler/tests/test_generated_code.py index 76f2739b55..ad64e54b5e 100644 --- a/compiler/fory_compiler/tests/test_generated_code.py +++ b/compiler/fory_compiler/tests/test_generated_code.py @@ -205,6 +205,37 @@ def test_rust_generated_code_can_use_chrono_temporal_types(): assert "::fory::Duration" not in rust_output +def test_rust_nested_container_ref_uses_correct_pointer_type(): + schema = parse_fdl( + dedent( + """ + package gen; + + message Node { + string value = 1; + } + + message Request { + list> groups = 1; + map> nodes = 2; + } + """ + ) + ) + + rust_output = render_files(generate_files(schema, RustGenerator)) + assert ( + "pub groups: ::std::vec::Vec<::std::vec::Vec<::std::sync::Arc>>" + in rust_output + ) + assert ( + "pub nodes: ::std::collections::HashMap<" + "::std::string::String, " + "::std::collections::HashMap<::std::string::String, ::std::sync::Arc>>" + in rust_output + ) + + def test_generated_code_integer_encoding_variants_equivalent(): fdl = dedent( """ diff --git a/docs/guide/java/type-registration.md b/docs/guide/java/type-registration.md index f05a7665db..15270c7ee9 100644 --- a/docs/guide/java/type-registration.md +++ b/docs/guide/java/type-registration.md @@ -40,6 +40,9 @@ fory.register(SomeClass1.class, 1); ``` Note that class registration order is important. Serialization and deserialization peers should have the same registration order. +Register classes and custom serializers before the first top-level `serialize`, `deserialize`, or +`copy` call on a `Fory` instance. Fory freezes registration at that point so runtime lookups can use +the finalized registration state. Internal type IDs 0-32 are reserved for built-in xlang types. Java native built-ins start at `Types.NONE + 1`, and user IDs are encoded as `(user_id << 8) | internal_type_id`. diff --git a/integration_tests/jpms_tests/src/test/java/org/apache/fory/integration_tests/JpmsFieldAccessorTest.java b/integration_tests/jpms_tests/src/test/java/org/apache/fory/integration_tests/JpmsFieldAccessorTest.java index c9414385ff..700f3a20bb 100644 --- a/integration_tests/jpms_tests/src/test/java/org/apache/fory/integration_tests/JpmsFieldAccessorTest.java +++ b/integration_tests/jpms_tests/src/test/java/org/apache/fory/integration_tests/JpmsFieldAccessorTest.java @@ -20,6 +20,10 @@ package org.apache.fory.integration_tests; import java.lang.reflect.Field; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; import org.apache.fory.Fory; import org.apache.fory.integration_tests.model.NonSerializableNoNoArgBean; import org.apache.fory.integration_tests.model.PrivateFieldBean; @@ -64,13 +68,36 @@ public void testNonSerializableNoNoArgSerialization() { Fory.builder().withXlang(false).withCodegen(false).requireClassRegistration(false).build(); byte[] bytes = fory.serialize(new NonSerializableNoNoArgBean(5, 13)); NonSerializableNoNoArgBean.resetParentConstructorCalls(); - NonSerializableNoNoArgBean result = - (NonSerializableNoNoArgBean) fory.deserialize(bytes); + NonSerializableNoNoArgBean result = (NonSerializableNoNoArgBean) fory.deserialize(bytes); Assert.assertEquals(result.parentValue(), 5); Assert.assertEquals(result.value(), 13); Assert.assertEquals(NonSerializableNoNoArgBean.parentConstructorCalls(), 0); } + @Test + public void testSetFromMapJdkFieldRestore() throws Exception { + if (JDK_MAJOR_VERSION < 25) { + return; + } + Fory fory = + Fory.builder().withXlang(false).withCodegen(false).requireClassRegistration(false).build(); + Map backingMap = Collections.synchronizedMap(new HashMap<>()); + Set set = Collections.newSetFromMap(backingMap); + set.add("alpha"); + set.add("beta"); + + Set result = (Set) fory.deserialize(fory.serialize(set)); + Assert.assertEquals(result, set); + Field mapField = result.getClass().getDeclaredField("m"); + Object restoredMap = FieldAccessor.createAccessor(mapField).getObject(result); + Assert.assertEquals(restoredMap.getClass().getName(), backingMap.getClass().getName()); + + Set copied = fory.copy(set); + Assert.assertEquals(copied, set); + Object copiedMap = FieldAccessor.createAccessor(mapField).getObject(copied); + Assert.assertEquals(copiedMap.getClass().getName(), backingMap.getClass().getName()); + } + @Test public void testCodegenFinalFieldAccess() throws Exception { if (JDK_MAJOR_VERSION < 25) { diff --git a/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/ForyStructProcessor.java b/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/ForyStructProcessor.java index 2eb7e7d496..33015c2959 100644 --- a/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/ForyStructProcessor.java +++ b/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/ForyStructProcessor.java @@ -376,6 +376,13 @@ private SourceField buildField( boolean serialized, SerializerMode mode) { Set modifiers = field.getModifiers(); + if (!record && modifiers.contains(Modifier.FINAL)) { + throw new InvalidStructException( + "Static serializers cannot assign final field " + + field.getSimpleName() + + "; use a record component or mark the field @Ignore/transient", + field); + } ForyFieldMeta foryField = foryField(field); Object fieldTypeTree = typeTree(field); boolean nullable = fieldNullable(field.asType(), fieldTypeTree, mode); @@ -390,7 +397,6 @@ private SourceField buildField( SourceField.AccessKind writeKind; String readAccess; String writeAccess; - boolean finalField = modifiers.contains(Modifier.FINAL); if (record) { readKind = SourceField.AccessKind.METHOD; writeKind = SourceField.AccessKind.METHOD; @@ -403,20 +409,19 @@ private SourceField buildField( writeAccess = readAccess; } else { ExecutableElement getter = findGetter(owner, field, generatedPackage); - ExecutableElement setter = finalField ? null : findSetter(owner, field, generatedPackage); - if (getter == null || (!finalField && setter == null)) { + ExecutableElement setter = findSetter(owner, field, generatedPackage); + if (getter == null || setter == null) { throw new InvalidStructException( "Field " + field.getSimpleName() + " is not directly accessible from the generated serializer. Add accessible " - + (finalField ? "non-private getter" : "non-private getter/setter") - + " methods or mark it @Ignore/transient.", + + "non-private getter/setter methods or mark it @Ignore/transient.", field); } readKind = SourceField.AccessKind.METHOD; writeKind = SourceField.AccessKind.METHOD; readAccess = getter.getSimpleName().toString(); - writeAccess = finalField ? null : setter.getSimpleName().toString(); + writeAccess = setter.getSimpleName().toString(); } return new SourceField( id, @@ -431,7 +436,6 @@ private SourceField buildField( readAccess, writeKind, writeAccess, - finalField, foryField.hasForyField, foryField.id, nullable, diff --git a/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/SourceField.java b/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/SourceField.java index e0d7b8a6b1..3d33d37422 100644 --- a/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/SourceField.java +++ b/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/SourceField.java @@ -37,7 +37,6 @@ enum AccessKind { final String readAccess; final AccessKind writeAccessKind; final String writeAccess; - final boolean finalField; final boolean hasForyField; final int foryFieldId; final boolean nullable; @@ -58,7 +57,6 @@ enum AccessKind { String readAccess, AccessKind writeAccessKind, String writeAccess, - boolean finalField, boolean hasForyField, int foryFieldId, boolean nullable, @@ -77,7 +75,6 @@ enum AccessKind { this.readAccess = readAccess; this.writeAccessKind = writeAccessKind; this.writeAccess = writeAccess; - this.finalField = finalField; this.hasForyField = hasForyField; this.foryFieldId = foryFieldId; this.nullable = nullable; diff --git a/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/StaticSerializerSourceWriter.java b/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/StaticSerializerSourceWriter.java index f439a2c140..ce52ad379d 100644 --- a/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/StaticSerializerSourceWriter.java +++ b/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/StaticSerializerSourceWriter.java @@ -320,11 +320,7 @@ private void writeReadBeanGroup( for (SourceField field : struct.fields) { builder.append(" case ").append(field.id).append(":\n"); if (canEmitDirectReadField(field)) { - if (field.finalField) { - appendFinalDirectRead(field, "value", "fieldInfo"); - } else { - appendDirectRead(field); - } + appendDirectRead(field); } else { String fieldValueName = "fieldValue" + field.id; if (hasDirectReadField()) { @@ -337,17 +333,10 @@ private void writeReadBeanGroup( } else { fieldValueName = "fieldValue"; } - builder.append(" "); - if (field.finalField) { - builder - .append("setGeneratedFieldValue(value, fieldInfo, ") - .append(field.castExpression(fieldValueName)) - .append(");\n"); - } else { - builder - .append(field.writeStatement("value", field.castExpression(fieldValueName))) - .append("\n"); - } + builder + .append(" ") + .append(field.writeStatement("value", field.castExpression(fieldValueName))) + .append("\n"); } builder.append(" break;\n"); } @@ -479,75 +468,6 @@ private void appendDirectRead(SourceField field) { builder.append(" ").append(field.writeStatement("value", exactRead)).append("\n"); } - private void appendFinalDirectRead(SourceField field, String targetName, String fieldInfoName) { - if (canEmitDirectStringField(field)) { - builder - .append(" setGeneratedFieldValue(") - .append(targetName) - .append(", ") - .append(fieldInfoName) - .append(", readContext.readString());\n"); - return; - } - if (canEmitDirectArrayField(field)) { - builder.append(" readContext.preserveRefId(-1);\n"); - builder - .append(" setGeneratedFieldValue(") - .append(targetName) - .append(", ") - .append(fieldInfoName) - .append(", ") - .append(field.castExpression("readContext.readNonRef(fieldInfo.typeInfo)")) - .append(");\n"); - return; - } - String exactRead = exactPrimitiveReadExpression(field); - if (exactRead == null) { - appendFinalPrimitiveReadSwitch(field, targetName, fieldInfoName); - return; - } - builder - .append(" setGeneratedFieldValue(") - .append(targetName) - .append(", ") - .append(fieldInfoName) - .append(", ") - .append(exactRead) - .append(");\n"); - } - - private void appendFinalPrimitiveReadSwitch( - SourceField field, String targetName, String fieldInfoName) { - builder.append(" switch (fieldInfo.dispatchId) {\n"); - String[][] cases = primitiveReadCases(field); - for (String[] readCase : cases) { - builder.append(" case DispatchId.").append(readCase[0]).append(":\n"); - builder - .append(" setGeneratedFieldValue(") - .append(targetName) - .append(", ") - .append(fieldInfoName) - .append(", ") - .append(readCase[1]) - .append(");\n"); - builder.append(" break;\n"); - } - builder.append(" default:\n"); - builder - .append(" Object fieldValue") - .append(field.id) - .append(" = readBuildInFieldValue(readContext, fieldInfo);\n"); - builder - .append(" setGeneratedFieldValue(") - .append(targetName) - .append(", ") - .append(fieldInfoName) - .append(", ") - .append(field.castExpression("fieldValue" + field.id)) - .append(");\n"); - builder.append(" }\n"); - } - private void appendPrimitiveReadSwitch(SourceField field) { builder.append(" switch (fieldInfo.dispatchId) {\n"); String[][] cases = primitiveReadCases(field); @@ -913,19 +833,10 @@ private void writeCompatibleBeanDispatchGroup(int group) { .append(field.id) .append("]);\n"); appendDebugRemoteRead("after read", "remoteField", 10); - builder.append(" "); - if (field.finalField) { - builder - .append("setGeneratedFieldValue(value, fieldsById[") - .append(field.id) - .append("], ") - .append(field.castExpression("fieldValue")) - .append(");\n"); - } else { - builder - .append(field.writeStatement("value", field.castExpression("fieldValue"))) - .append("\n"); - } + builder + .append(" ") + .append(field.writeStatement("value", field.castExpression("fieldValue"))) + .append("\n"); builder.append(" } else {\n"); appendDebugRemoteRead("before skip", "remoteField", 10); builder.append(" skipField(readContext, remoteField);\n"); @@ -1085,23 +996,17 @@ private void writeCopy() { builder.append(" ").append(struct.typeName).append(" copied = newBean();\n"); builder.append(" copyContext.reference(value, copied);\n"); for (SourceField field : struct.fields) { - String copiedExpression = - field.castExpression( - "copyFieldValue(copyContext, " - + field.readExpression("value") - + ", fieldsById[" - + field.id - + "])"); builder .append(" ") .append( - field.finalField - ? "setGeneratedFieldValue(copied, fieldsById[" - + field.id - + "], " - + copiedExpression - + ");" - : field.writeStatement("copied", copiedExpression)) + field.writeStatement( + "copied", + field.castExpression( + "copyFieldValue(copyContext, " + + field.readExpression("value") + + ", fieldsById[" + + field.id + + "])"))) .append("\n"); } builder.append(" return copied;\n"); diff --git a/java/fory-annotation-processor/src/test/java/org/apache/fory/annotation/processing/ForyStructProcessorTest.java b/java/fory-annotation-processor/src/test/java/org/apache/fory/annotation/processing/ForyStructProcessorTest.java index c47ef56402..2e4092e14e 100644 --- a/java/fory-annotation-processor/src/test/java/org/apache/fory/annotation/processing/ForyStructProcessorTest.java +++ b/java/fory-annotation-processor/src/test/java/org/apache/fory/annotation/processing/ForyStructProcessorTest.java @@ -115,54 +115,6 @@ public void testLegacyBooleanEvolvingAnnotationCompiles() throws Exception { } } - @Test - public void testStaticFinalFields() throws Exception { - CompilationResult result = - compile( - "test.FinalFieldStruct", - "package test;\n" - + "import org.apache.fory.annotation.ForyStruct;\n" - + "@ForyStruct public class FinalFieldStruct {\n" - + " private final String name;\n" - + " private final int age;\n" - + " public String note;\n" - + " public FinalFieldStruct(String name, int age) {\n" - + " this.name = name;\n" - + " this.age = age;\n" - + " }\n" - + " public String getName() { return name; }\n" - + " public int getAge() { return age; }\n" - + "}\n"); - Assert.assertTrue(result.success, result.diagnostics()); - String generatedSource = - result.generatedSource("test/FinalFieldStruct_ForyNativeSerializer.java"); - Assert.assertFalse(generatedSource.contains("constructorFieldBits"), generatedSource); - Assert.assertTrue(generatedSource.contains("setGeneratedFieldValue"), generatedSource); - try (URLClassLoader loader = result.classLoader()) { - Class type = loader.loadClass("test.FinalFieldStruct"); - Object value = type.getConstructor(String.class, int.class).newInstance("fory", 12); - setField(type, value, "note", "static"); - Fory fory = - Fory.builder() - .withXlang(false) - .withClassLoader(loader) - .withCodegen(false) - .requireClassRegistration(false) - .build(); - Object serializer = fory.getTypeResolver().getTypeInfo(type).getSerializer(); - Assert.assertTrue(serializer instanceof StaticGeneratedStructSerializer); - - Object roundTrip = fory.deserialize(fory.serialize(value)); - Assert.assertEquals(invoke(type, roundTrip, "getName"), "fory"); - Assert.assertEquals(invoke(type, roundTrip, "getAge"), 12); - Assert.assertEquals(getField(type, roundTrip, "note"), "static"); - Object copied = fory.copy(value); - Assert.assertEquals(invoke(type, copied, "getName"), "fory"); - Assert.assertEquals(invoke(type, copied, "getAge"), 12); - Assert.assertEquals(getField(type, copied, "note"), "static"); - } - } - @Test public void testForyDebugAnnotationEmitsGeneratedFieldTracing() throws Exception { CompilationResult result = diff --git a/java/fory-core/src/main/java/org/apache/fory/builder/StaticCompatibleCodecBuilder.java b/java/fory-core/src/main/java/org/apache/fory/builder/StaticCompatibleCodecBuilder.java index 12a3631627..8173b15495 100644 --- a/java/fory-core/src/main/java/org/apache/fory/builder/StaticCompatibleCodecBuilder.java +++ b/java/fory-core/src/main/java/org/apache/fory/builder/StaticCompatibleCodecBuilder.java @@ -59,6 +59,9 @@ */ public final class StaticCompatibleCodecBuilder extends ObjectCodecBuilder { private static final int DISPATCH_GROUP_SIZE = 8; + // Hidden generated serializers cannot use private split helpers because Janino emits private + // self-invokes against the source binary name, which Lookup#defineHiddenClass cannot resolve. + private static final String DISPATCH_METHOD_MODIFIERS = "final"; private final List localDescriptors; private final boolean debug; @@ -234,7 +237,7 @@ private void genDispatchMethods() { int groupCount = (localDescriptors.size() + DISPATCH_GROUP_SIZE - 1) / DISPATCH_GROUP_SIZE; if (isRecord) { ctx.addMethod( - "private", + DISPATCH_METHOD_MODIFIERS, "readMatchedRecordField", genDispatchRouter("readMatchedRecordField", groupCount), void.class, @@ -246,7 +249,7 @@ private void genDispatchMethods() { "_f_remoteField"); for (int group = 0; group < groupCount; group++) { ctx.addMethod( - "private", + DISPATCH_METHOD_MODIFIERS, "readMatchedRecordField" + group, genRecordDispatchGroup(group), void.class, @@ -260,7 +263,7 @@ private void genDispatchMethods() { return; } ctx.addMethod( - "private", + DISPATCH_METHOD_MODIFIERS, "readMatchedField", genDispatchRouter("readMatchedField", groupCount), void.class, @@ -272,7 +275,7 @@ private void genDispatchMethods() { "_f_remoteField"); for (int group = 0; group < groupCount; group++) { ctx.addMethod( - "private", + DISPATCH_METHOD_MODIFIERS, "readMatchedField" + group, genObjectDispatchGroup(group, valueTypeRef), void.class, diff --git a/java/fory-core/src/main/java/org/apache/fory/platform/GraalvmSupport.java b/java/fory-core/src/main/java/org/apache/fory/platform/GraalvmSupport.java index 573b379f83..aa119f2e60 100644 --- a/java/fory-core/src/main/java/org/apache/fory/platform/GraalvmSupport.java +++ b/java/fory-core/src/main/java/org/apache/fory/platform/GraalvmSupport.java @@ -118,6 +118,7 @@ public class GraalvmSupport { registerDefaultSerializerClass(CollectionSerializers.DefaultJavaCollectionSerializer.class); registerDefaultSerializerClass(CollectionSerializer.class); registerDefaultSerializerClass(MapSerializers.JDKCompatibleMapSerializer.class); + registerDefaultSerializerClass(MapSerializers.IdentityHashMapSerializer.class); registerDefaultSerializerClass(MapSerializers.DefaultJavaMapSerializer.class); registerDefaultSerializerClass(MapSerializer.class); registerDefaultSerializerClass(CodegenSerializer.LazyInitBeanSerializer.class); diff --git a/java/fory-core/src/main/java/org/apache/fory/platform/internal/DefineClass.java b/java/fory-core/src/main/java/org/apache/fory/platform/internal/DefineClass.java index 3b57105a41..4ea20ae703 100644 --- a/java/fory-core/src/main/java/org/apache/fory/platform/internal/DefineClass.java +++ b/java/fory-core/src/main/java/org/apache/fory/platform/internal/DefineClass.java @@ -144,12 +144,11 @@ private static RuntimeException hiddenClassFailure(Class neighbor, Throwable if (cause instanceof Error) { throw (Error) cause; } - return new IllegalStateException( - "Cannot define hidden nestmate for " - + neighbor.getName() - + ". JDK25+ generated serializers require java.base/java.lang.invoke to be open to " - + "org.apache.fory.core.", - cause); + String message = "Cannot define hidden nestmate for " + neighbor.getName() + "."; + if (JdkVersion.MAJOR_VERSION >= 25) { + message += " " + _JDKAccess.jdk25AccessMessage(); + } + return new IllegalStateException(message, cause); } private static final class HiddenClassDefiner { diff --git a/java/fory-core/src/main/java/org/apache/fory/platform/internal/_JDKAccess.java b/java/fory-core/src/main/java/org/apache/fory/platform/internal/_JDKAccess.java index 4d67f0b193..c989207e2c 100644 --- a/java/fory-core/src/main/java/org/apache/fory/platform/internal/_JDKAccess.java +++ b/java/fory-core/src/main/java/org/apache/fory/platform/internal/_JDKAccess.java @@ -100,6 +100,13 @@ public static Lookup _trustedLookup(Class objectClass) { return lookupCache.get(objectClass, () -> _Lookup._trustedLookup(objectClass)); } + public static String jdk25AccessMessage() { + return "JDK25 zero-Unsafe mode requires java.base/java.lang.invoke to be open to Fory. " + + "Use --add-opens=java.base/java.lang.invoke=ALL-UNNAMED when Fory is on the " + + "classpath, or --add-opens=java.base/java.lang.invoke=org.apache.fory.core when " + + "Fory is on the module path."; + } + public static T tryMakeFunction( Lookup lookup, MethodHandle handle, Class functionInterface) { try { diff --git a/java/fory-core/src/main/java/org/apache/fory/reflect/ObjectInstantiators.java b/java/fory-core/src/main/java/org/apache/fory/reflect/ObjectInstantiators.java index 7f2925c5cf..4043d63692 100644 --- a/java/fory-core/src/main/java/org/apache/fory/reflect/ObjectInstantiators.java +++ b/java/fory-core/src/main/java/org/apache/fory/reflect/ObjectInstantiators.java @@ -117,7 +117,7 @@ public static ObjectInstantiator createObjectInstantiator(Class type) } if (noArgConstructor == null) { if (JdkVersion.MAJOR_VERSION >= 25) { - return new ParentNoArgCtrInstantiator<>(type); + return new ReflectionFactoryInstantiator<>(type); } return new UnsafeObjectInstantiator<>(type); } @@ -337,6 +337,48 @@ private static MethodHandle newInstanceHandle() { } } + static final class ReflectionFactoryInstantiator extends ObjectInstantiator { + private static final Constructor OBJECT_CONSTRUCTOR = objectConstructor(); + private final Constructor constructor; + + ReflectionFactoryInstantiator(Class type) { + super(type); + constructor = createBypassConstructor(type); + } + + private static Constructor objectConstructor() { + try { + return Object.class.getDeclaredConstructor(); + } catch (NoSuchMethodException e) { + throw new ExceptionInInitializerError(e); + } + } + + private static Constructor createBypassConstructor(Class type) { + try { + return (Constructor) + ReflectionFactoryAccess.newConstructorForSerialization(type, OBJECT_CONSTRUCTOR); + } catch (Throwable e) { + throw new ForyException( + "Failed to create constructor-bypassing instantiator for " + type, e); + } + } + + @Override + public T newInstance() { + try { + return constructor.newInstance(); + } catch (Exception e) { + throw makeException(type, e); + } + } + + @Override + public T newInstanceWithArguments(Object... arguments) { + throw new UnsupportedOperationException(); + } + } + public static final class ParentNoArgCtrInstantiator extends ObjectInstantiator { private final Constructor constructor; @@ -359,10 +401,7 @@ private static Constructor createSerializationConstructor(Class type) private static Constructor findSerializationConstructor(Class type) throws NoSuchMethodException { if (!Serializable.class.isAssignableFrom(type)) { - // ReflectionFactory can synthesize serialization constructors for ordinary classes too. - // Use Object as the template so normal Fory object creation keeps empty-instance - // semantics and never depends on the target class hierarchy exposing a no-arg constructor. - return Object.class.getDeclaredConstructor(); + throw new ForyException("ObjectStream instantiation requires Serializable type " + type); } Class current = type.getSuperclass(); // Java ObjectStream reconstruction skips every Serializable class constructor and invokes @@ -409,43 +448,43 @@ public T newInstance() { public T newInstanceWithArguments(Object... arguments) { throw new UnsupportedOperationException(); } + } - private static final class ReflectionFactoryAccess { - private static final Object REFLECTION_FACTORY; - private static final MethodHandle NEW_CONSTRUCTOR_FOR_SERIALIZATION; + private static final class ReflectionFactoryAccess { + private static final Object REFLECTION_FACTORY; + private static final MethodHandle NEW_CONSTRUCTOR_FOR_SERIALIZATION; - static { - try { - Class reflectionFactoryClass = reflectionFactoryClass(); - MethodHandles.Lookup lookup = _JDKAccess._trustedLookup(reflectionFactoryClass); - MethodHandle getReflectionFactory = - lookup.findStatic( - reflectionFactoryClass, - "getReflectionFactory", - MethodType.methodType(reflectionFactoryClass)); - REFLECTION_FACTORY = getReflectionFactory.invoke(); - NEW_CONSTRUCTOR_FOR_SERIALIZATION = - lookup.findVirtual( - reflectionFactoryClass, - "newConstructorForSerialization", - MethodType.methodType(Constructor.class, Class.class, Constructor.class)); - } catch (Throwable e) { - throw new ExceptionInInitializerError(e); - } + static { + try { + Class reflectionFactoryClass = reflectionFactoryClass(); + MethodHandles.Lookup lookup = _JDKAccess._trustedLookup(reflectionFactoryClass); + MethodHandle getReflectionFactory = + lookup.findStatic( + reflectionFactoryClass, + "getReflectionFactory", + MethodType.methodType(reflectionFactoryClass)); + REFLECTION_FACTORY = getReflectionFactory.invoke(); + NEW_CONSTRUCTOR_FOR_SERIALIZATION = + lookup.findVirtual( + reflectionFactoryClass, + "newConstructorForSerialization", + MethodType.methodType(Constructor.class, Class.class, Constructor.class)); + } catch (Throwable e) { + throw new ExceptionInInitializerError(e); } + } - private static Class reflectionFactoryClass() throws ClassNotFoundException { - if (JdkVersion.MAJOR_VERSION >= 25) { - return Class.forName("jdk.internal.reflect.ReflectionFactory"); - } - return Class.forName("sun.reflect.ReflectionFactory"); + private static Class reflectionFactoryClass() throws ClassNotFoundException { + if (JdkVersion.MAJOR_VERSION >= 25) { + return Class.forName("jdk.internal.reflect.ReflectionFactory"); } + return Class.forName("sun.reflect.ReflectionFactory"); + } - private static Constructor newConstructorForSerialization( - Class type, Constructor parentConstructor) throws Throwable { - return (Constructor) - NEW_CONSTRUCTOR_FOR_SERIALIZATION.invoke(REFLECTION_FACTORY, type, parentConstructor); - } + private static Constructor newConstructorForSerialization( + Class type, Constructor parentConstructor) throws Throwable { + return (Constructor) + NEW_CONSTRUCTOR_FOR_SERIALIZATION.invoke(REFLECTION_FACTORY, type, parentConstructor); } } } diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java index 1159216721..a23f2d20ab 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java @@ -50,6 +50,7 @@ import java.util.EnumSet; import java.util.HashMap; import java.util.HashSet; +import java.util.IdentityHashMap; import java.util.LinkedHashMap; import java.util.LinkedList; import java.util.List; @@ -1519,6 +1520,9 @@ public Class getSerializerClass(Class cls, boolean code if (serializerClass != null) { return serializerClass; } + if (!isCrossLanguage() && cls == IdentityHashMap.class) { + return MapSerializers.IdentityHashMapSerializer.class; + } if (Externalizable.class.isAssignableFrom(cls) || requireJavaSerialization(cls) || useReplaceResolveSerializer(cls)) { diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/ExceptionSerializers.java b/java/fory-core/src/main/java/org/apache/fory/serializer/ExceptionSerializers.java index ae23e3469b..e5689087ff 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/ExceptionSerializers.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/ExceptionSerializers.java @@ -81,8 +81,8 @@ && hasSubclassFields(slotsSerializers)) { throw new ForyException( "Throwable serialization for JDK type " + type.getName() - + " with subclass fields requires JDK internal field access. On JDK25+, open " - + "java.base/java.lang.invoke to org.apache.fory.core."); + + " with subclass fields requires JDK internal field access. " + + jdkFieldAccessMessage()); } // Native-image runtime must rebuild slot serializers once so field accessors and // descriptors are created against the runtime heap layout instead of reusing @@ -140,7 +140,7 @@ private T readAndroidThrowableWithoutDetailMessageField( "Deserializing Throwable type " + type.getName() + " without a String message constructor requires JDK internal field access. " - + "On JDK25+, open java.base/java.lang.invoke to org.apache.fory.core."); + + jdkFieldAccessMessage()); } int refId = readContext.lastPreservedRefId(); if (refId >= 0) { @@ -154,8 +154,8 @@ private T readAndroidThrowableWithoutDetailMessageField( throw new ForyException( "Deserializing cyclic Throwable references for type " + type.getName() - + " requires JDK internal field access. On JDK25+, open java.base/java.lang.invoke " - + "to org.apache.fory.core."); + + " requires JDK internal field access. " + + jdkFieldAccessMessage()); } T obj = newThrowableWithMessage(detailMessage); readContext.reference(obj); @@ -223,6 +223,13 @@ private void writeNumClassLayers(MemoryBuffer buffer, Serializer[] slotsSerializ } } + private static String jdkFieldAccessMessage() { + if (!AndroidSupport.IS_ANDROID && JdkVersion.MAJOR_VERSION >= 25) { + return _JDKAccess.jdk25AccessMessage(); + } + return "This Throwable shape is unsupported on runtimes without JDK internal field access."; + } + public static final class StackTraceElementSerializer extends Serializer { private static final MethodHandles.Lookup LOOKUP = AndroidSupport.IS_ANDROID diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/CollectionSerializers.java b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/CollectionSerializers.java index e1ea4b0217..e87bf27298 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/CollectionSerializers.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/CollectionSerializers.java @@ -57,7 +57,6 @@ import org.apache.fory.exception.ForyException; import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.memory.MemoryUtils; -import org.apache.fory.platform.JdkVersion; import org.apache.fory.reflect.FieldAccessor; import org.apache.fory.reflect.ReflectionUtils; import org.apache.fory.resolver.ClassResolver; @@ -579,32 +578,27 @@ public Collection newCollection(CopyContext copyContext, Collection originCollec public static final class SetFromMapSerializer extends CollectionSerializer> { private static final List EMPTY_COLLECTION_STUB = new ArrayList<>(); - private static final class JvmSetFromMapAccess { + private static final class SetFromMapAccess { private static final FieldAccessor MAP_ACCESSOR; + private static final FieldAccessor KEY_SET_ACCESSOR; static { try { Class type = Class.forName("java.util.Collections$SetFromMap"); - Field mapField = type.getDeclaredField("m"); - MAP_ACCESSOR = FieldAccessor.createAccessor(mapField); + MAP_ACCESSOR = FieldAccessor.createAccessor(type.getDeclaredField("m")); + KEY_SET_ACCESSOR = FieldAccessor.createAccessor(type.getDeclaredField("s")); } catch (final Exception e) { throw new RuntimeException(e); } } - } - private static final class LegacySetFromMapAccess { - private static final FieldAccessor M_ACCESSOR; - private static final FieldAccessor S_ACCESSOR; + static Map map(Set set) { + return (Map) MAP_ACCESSOR.getObject(set); + } - static { - try { - Class type = Class.forName("java.util.Collections$SetFromMap"); - M_ACCESSOR = FieldAccessor.createAccessor(type.getDeclaredField("m")); - S_ACCESSOR = FieldAccessor.createAccessor(type.getDeclaredField("s")); - } catch (final Exception e) { - throw new RuntimeException(e); - } + static void restore(Set set, Map map) { + MAP_ACCESSOR.putObject(set, map); + KEY_SET_ACCESSOR.putObject(set, map.keySet()); } } @@ -627,17 +621,17 @@ public Collection newCollection(ReadContext readContext) { } else { if (!MemoryUtils.JDK_COLLECTION_FIELD_ACCESS) { throw new UnsupportedOperationException( - "This runtime cannot read legacy SetFromMap payloads that require hidden JDK field " + "This runtime cannot read SetFromMap backing-map payloads that require hidden JDK field " + "restoration"); } - Map map = (Map) mapSerializer.read(readContext); + Map map = (Map) mapSerializer.read(readContext); try { set = Collections.newSetFromMap(new HashMap<>()); - LegacySetFromMapAccess.M_ACCESSOR.putObject(set, map); - LegacySetFromMapAccess.S_ACCESSOR.putObject(set, map.keySet()); + SetFromMapAccess.restore(set, map); } catch (Throwable e) { throw new UnsupportedOperationException( - "This runtime cannot restore legacy SetFromMap payloads through final JDK fields", e); + "This runtime cannot restore SetFromMap backing-map payloads through final JDK fields", + e); } setNumElements(0); } @@ -648,21 +642,20 @@ public Collection newCollection(ReadContext readContext) { @Override public Collection newCollection(CopyContext copyContext, Collection originCollection) { assert !config.isXlang(); + return Collections.newSetFromMap(new HashMap(originCollection.size())); + } + + @Override + public Set copy(CopyContext copyContext, Set originCollection) { if (!MemoryUtils.JDK_COLLECTION_FIELD_ACCESS) { - if (JdkVersion.MAJOR_VERSION >= 25) { - throw unsupportedJdk25SetFromMap(null); - } - return Collections.newSetFromMap(new HashMap(originCollection.size())); - } - Map map = - (Map) JvmSetFromMapAccess.MAP_ACCESSOR.getObject(originCollection); - MapLikeSerializer mapSerializer = - (MapLikeSerializer) typeResolver.getSerializer(map.getClass()); - if (JdkVersion.MAJOR_VERSION >= 25 && !mapSerializer.supportCodegenHook) { - throw unsupportedJdk25SetFromMap(map.getClass()); + return (Set) super.copy(copyContext, originCollection); } - Map newMap = mapSerializer.newMap(copyContext, map); - return Collections.newSetFromMap(newMap); + Map map = SetFromMapAccess.map(originCollection); + Set result = Collections.newSetFromMap(new HashMap<>()); + copyContext.reference(originCollection, result); + Map newMap = copyContext.copyObject(map); + SetFromMapAccess.restore(result, newMap); + return result; } @Override @@ -671,9 +664,6 @@ public Collection onCollectionWrite(WriteContext writeContext, Set value) { Map map; TypeInfo typeInfo; if (!MemoryUtils.JDK_COLLECTION_FIELD_ACCESS) { - if (JdkVersion.MAJOR_VERSION >= 25) { - throw unsupportedJdk25SetFromMap(null); - } HashMap source = new HashMap<>(value.size()); for (Object element : value) { source.put(element, Boolean.TRUE); @@ -681,15 +671,10 @@ public Collection onCollectionWrite(WriteContext writeContext, Set value) { map = source; typeInfo = typeResolver.getTypeInfo(HashMap.class); } else { - map = (Map) JvmSetFromMapAccess.MAP_ACCESSOR.getObject(value); + map = SetFromMapAccess.map(value); typeInfo = typeResolver.getTypeInfo(map.getClass()); } MapLikeSerializer mapSerializer = (MapLikeSerializer) typeInfo.getSerializer(); - // The legacy payload restores Collections$SetFromMap by writing its final JDK fields. - // JDK25 zero-Unsafe mode cannot do that, so unsupported backing maps must fail before write. - if (JdkVersion.MAJOR_VERSION >= 25 && !mapSerializer.supportCodegenHook) { - throw unsupportedJdk25SetFromMap(map.getClass()); - } typeResolver.writeTypeInfo(writeContext, typeInfo); if (mapSerializer.supportCodegenHook) { buffer.writeBoolean(true); @@ -701,16 +686,6 @@ public Collection onCollectionWrite(WriteContext writeContext, Set value) { return EMPTY_COLLECTION_STUB; } } - - private static UnsupportedOperationException unsupportedJdk25SetFromMap(Class mapType) { - String mapDescription = mapType == null ? "an inaccessible backing map" : mapType.getName(); - return new UnsupportedOperationException( - "JDK25+ zero-Unsafe mode cannot serialize Collections.newSetFromMap backed by " - + mapDescription - + " because that would require hidden final JDK field restoration. Use a backing " - + "map with public-constructor serialization support or register a custom " - + "serializer."); - } } public static final class ConcurrentHashMapKeySetViewSerializer diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/MapSerializers.java b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/MapSerializers.java index ceaab81a3d..dd1b754e3e 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/MapSerializers.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/MapSerializers.java @@ -34,6 +34,7 @@ import java.util.Comparator; import java.util.EnumMap; import java.util.HashMap; +import java.util.IdentityHashMap; import java.util.LinkedHashMap; import java.util.Map; import java.util.Map.Entry; @@ -119,6 +120,24 @@ public Map newMap(Map map) { } } + public static final class IdentityHashMapSerializer + extends JDKCompatibleMapSerializer { + public IdentityHashMapSerializer(TypeResolver typeResolver) { + super(typeResolver, IdentityHashMap.class); + } + + @Override + public IdentityHashMap copy(CopyContext copyContext, IdentityHashMap value) { + IdentityHashMap copy = new IdentityHashMap(value.size()); + copyContext.reference(value, copy); + for (Object entryObject : value.entrySet()) { + Entry entry = (Entry) entryObject; + copy.put(copyContext.copyObject(entry.getKey()), copyContext.copyObject(entry.getValue())); + } + return copy; + } + } + public static final class LazyMapSerializer extends MapSerializer { public LazyMapSerializer(TypeResolver typeResolver) { super(typeResolver, LazyMap.class, true); diff --git a/java/fory-core/src/main/java/org/apache/fory/type/TypeUtils.java b/java/fory-core/src/main/java/org/apache/fory/type/TypeUtils.java index 58b01a8353..ab6adc373b 100644 --- a/java/fory-core/src/main/java/org/apache/fory/type/TypeUtils.java +++ b/java/fory-core/src/main/java/org/apache/fory/type/TypeUtils.java @@ -872,9 +872,8 @@ public static boolean isBean(TypeRef typeRef, TypeResolutionContext ctx) { || ctx.getCustomTypeRegistry().isExtraSupportedType(typeRef)) { return false; } - // Bean shape detection is independent of the later object-creation strategy. Construction may - // use a no-arg constructor, record/constructor-field metadata, or the platform empty-instance - // owner selected by TypeResolver. + // Bean shape detection is independent of the later object-creation strategy. Records use the + // record path; ordinary beans are created later by TypeResolver's object-instantiator owner. // bean class can be static nested class, but can't be not a non-static inner class if (cls.getEnclosingClass() != null && !Modifier.isStatic(cls.getModifiers())) { return false; diff --git a/java/fory-core/src/main/java25/org/apache/fory/memory/MemoryBuffer.java b/java/fory-core/src/main/java25/org/apache/fory/memory/MemoryBuffer.java index fef425a5fa..4e9e410e7e 100644 --- a/java/fory-core/src/main/java25/org/apache/fory/memory/MemoryBuffer.java +++ b/java/fory-core/src/main/java25/org/apache/fory/memory/MemoryBuffer.java @@ -814,7 +814,7 @@ public byte getByte(int index) { } // In the Java25 overlay, `_unsafe*` preserves the root MemoryBuffer unchecked-access naming. - // These methods use indexed array, ByteBuffer, and VarHandle access, not sun.misc.Unsafe. + // These methods use indexed array, ByteBuffer, and VarHandle access, not the JDK Unsafe API. // CHECKSTYLE.OFF:MethodName public byte _unsafeGetByte(int index) { // CHECKSTYLE.ON:MethodName diff --git a/java/fory-core/src/main/java25/org/apache/fory/platform/internal/_Lookup.java b/java/fory-core/src/main/java25/org/apache/fory/platform/internal/_Lookup.java index 2e4fa53562..e1c5878e9b 100644 --- a/java/fory-core/src/main/java25/org/apache/fory/platform/internal/_Lookup.java +++ b/java/fory-core/src/main/java25/org/apache/fory/platform/internal/_Lookup.java @@ -108,7 +108,6 @@ private static Lookup implLookup() { } private static String trustedLookupMessage() { - return "JDK25 zero-Unsafe mode requires java.base/java.lang.invoke to be open to " - + "org.apache.fory.core"; + return _JDKAccess.jdk25AccessMessage(); } } diff --git a/java/fory-core/src/main/java25/org/apache/fory/reflect/InstanceFieldAccessors.java b/java/fory-core/src/main/java25/org/apache/fory/reflect/InstanceFieldAccessors.java index 7963b1b221..022ed59f46 100644 --- a/java/fory-core/src/main/java25/org/apache/fory/reflect/InstanceFieldAccessors.java +++ b/java/fory-core/src/main/java25/org/apache/fory/reflect/InstanceFieldAccessors.java @@ -88,10 +88,7 @@ private static VarHandle fieldHandle(Field field) { private static IllegalStateException accessFailure(Field field, Throwable cause) { return new IllegalStateException( - "Cannot access field " - + field - + ". JDK25 zero-Unsafe mode requires java.base/java.lang.invoke to be open " - + "to org.apache.fory.core", + "Cannot access field " + field + ". " + _JDKAccess.jdk25AccessMessage(), cause); } diff --git a/java/fory-core/src/main/java25/org/apache/fory/serializer/PlatformStringUtils.java b/java/fory-core/src/main/java25/org/apache/fory/serializer/PlatformStringUtils.java index 5f347dad39..b399206bd1 100644 --- a/java/fory-core/src/main/java25/org/apache/fory/serializer/PlatformStringUtils.java +++ b/java/fory-core/src/main/java25/org/apache/fory/serializer/PlatformStringUtils.java @@ -91,8 +91,7 @@ private static StringHandles stringHandles() { : stringLookup.findVarHandle(String.class, "offset", int.class)); } catch (Throwable e) { throw new IllegalStateException( - "JDK25+ string internals require java.base/java.lang.invoke to be open to " - + "org.apache.fory.core", + "JDK25+ string internals are inaccessible. " + _JDKAccess.jdk25AccessMessage(), e); } } catch (NoSuchFieldException e) { diff --git a/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties b/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties index af974f69da..9fca2c8305 100644 --- a/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties +++ b/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties @@ -466,9 +466,11 @@ Args=--initialize-at-build-time=org.apache.fory.annotation.ForyField$Dynamic,\ org.apache.fory.serializer.collection.MapSerializers$SingletonMapSerializer,\ org.apache.fory.serializer.collection.MapSerializers$SortedMapSerializer,\ org.apache.fory.serializer.collection.MapSerializers$XlangMapSerializer,\ + org.apache.fory.serializer.collection.SynchronizedSerializers$SourceAccessors,\ org.apache.fory.serializer.collection.SynchronizedSerializers$SynchronizedCollectionSerializer,\ org.apache.fory.serializer.collection.SynchronizedSerializers$SynchronizedMapSerializer,\ org.apache.fory.serializer.collection.SynchronizedSerializers,\ + org.apache.fory.serializer.collection.UnmodifiableSerializers$SourceAccessors,\ org.apache.fory.serializer.collection.UnmodifiableSerializers$UnmodifiableCollectionSerializer,\ org.apache.fory.serializer.collection.UnmodifiableSerializers$UnmodifiableMapSerializer,\ org.apache.fory.serializer.collection.UnmodifiableSerializers,\ diff --git a/java/fory-core/src/test/java/org/apache/fory/ForyTest.java b/java/fory-core/src/test/java/org/apache/fory/ForyTest.java index 1832730930..7928fdebb1 100644 --- a/java/fory-core/src/test/java/org/apache/fory/ForyTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/ForyTest.java @@ -108,6 +108,33 @@ public void testReverseComparatorSerializer() { Assert.assertSame(fory.copy(comparator), comparator); } + @Test + public void testRegistrationFreezesOnUse() { + byte[] bytes = newNativeFory().serialize(1); + + Fory writer = newNativeFory(); + writer.serialize(1); + assertRegistrationFrozen(writer); + + Fory reader = newNativeFory(); + reader.deserialize(bytes); + assertRegistrationFrozen(reader); + + Fory copier = newNativeFory(); + copier.copy(1); + assertRegistrationFrozen(copier); + } + + private static Fory newNativeFory() { + return Fory.builder().withXlang(false).requireClassRegistration(false).build(); + } + + private static void assertRegistrationFrozen(Fory fory) { + assertThrows(ForyException.class, () -> fory.register(BeanA.class)); + assertThrows( + ForyException.class, () -> fory.registerSerializer(BeanA.class, ObjectSerializer.class)); + } + @Test(dataProvider = "crossLanguageReferenceTrackingConfig") public void primitivesTest(boolean referenceTracking, boolean xlang) { Fory fory1 = diff --git a/java/fory-core/src/test/java/org/apache/fory/reflect/ObjectInstantiatorsTest.java b/java/fory-core/src/test/java/org/apache/fory/reflect/ObjectInstantiatorsTest.java index dd4f12f5e4..3153913503 100644 --- a/java/fory-core/src/test/java/org/apache/fory/reflect/ObjectInstantiatorsTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/reflect/ObjectInstantiatorsTest.java @@ -27,8 +27,7 @@ import org.apache.fory.TestUtils; import org.apache.fory.exception.ForyException; import org.apache.fory.platform.AndroidSupport; -import org.apache.fory.platform.JdkVersion; -import org.apache.fory.reflect.ObjectInstantiators.ParentNoArgCtrInstantiator; +import org.apache.fory.reflect.ObjectInstantiators.ReflectionFactoryInstantiator; import org.testng.Assert; import org.testng.annotations.Test; @@ -64,25 +63,19 @@ static class NonSerializableChildWithoutNoArg extends NonSerializableParentWitho @Test public void testObjectInstantiator() { - if (JdkVersion.MAJOR_VERSION >= 25) { - return; - } - ParentNoArgCtrInstantiator instantiator = - new ParentNoArgCtrInstantiator<>(ArrayBlockingQueue.class); + ReflectionFactoryInstantiator instantiator = + new ReflectionFactoryInstantiator<>(ArrayBlockingQueue.class); Assert.assertEquals(instantiator.newInstance().getClass(), ArrayBlockingQueue.class); Assert.assertEquals( - new ParentNoArgCtrInstantiator<>(NoCtrTestClass.class).newInstance().getClass(), + new ReflectionFactoryInstantiator<>(NoCtrTestClass.class).newInstance().getClass(), NoCtrTestClass.class); } @Test public void testNonSerializableInstantiator() { - if (JdkVersion.MAJOR_VERSION >= 25) { - return; - } NonSerializableParentWithoutNoArg.constructorCalls = 0; - ParentNoArgCtrInstantiator instantiator = - new ParentNoArgCtrInstantiator<>(NonSerializableChildWithoutNoArg.class); + ReflectionFactoryInstantiator instantiator = + new ReflectionFactoryInstantiator<>(NonSerializableChildWithoutNoArg.class); NonSerializableChildWithoutNoArg instance = instantiator.newInstance(); Assert.assertEquals(instance.getClass(), NonSerializableChildWithoutNoArg.class); Assert.assertEquals(NonSerializableParentWithoutNoArg.constructorCalls, 0); diff --git a/java/fory-core/src/test/java/org/apache/fory/reflect/ReflectionUtilsTest.java b/java/fory-core/src/test/java/org/apache/fory/reflect/ReflectionUtilsTest.java index 6f8d862ec4..641659ce3d 100644 --- a/java/fory-core/src/test/java/org/apache/fory/reflect/ReflectionUtilsTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/reflect/ReflectionUtilsTest.java @@ -113,18 +113,8 @@ public NoArgConstructor1(int f1) { } @Test - public void testGetNoArgConstructor() throws Throwable { + public void testGetNoArgConstructor() { Constructor ctr = ReflectionUtils.getNoArgConstructor(NoArgConstructor1.class); Assert.assertNull(ctr); - - // ReflectionFactory serialization constructors are invoked directly by - // ParentNoArgCtrInstantiator. - // MethodHandle handle = lookup.unreflectConstructor(ctr); - // System.out.println(ctr); - // System.out.println(handle); - // org.apache.fory.reflect.ReflectionUtilsTest$NoArgConstructor1@4d339552 - // System.out.println(ctr.newInstance()); - // java.lang.Object@f0f2775 - // System.out.println(handle.invoke()); } } diff --git a/java/fory-core/src/test/java/org/apache/fory/serializer/ObjectSerializerTest.java b/java/fory-core/src/test/java/org/apache/fory/serializer/ObjectSerializerTest.java index b505cda775..72d4ec560b 100644 --- a/java/fory-core/src/test/java/org/apache/fory/serializer/ObjectSerializerTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/serializer/ObjectSerializerTest.java @@ -25,6 +25,7 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; +import java.io.Serializable; import java.nio.charset.StandardCharsets; import lombok.Data; import org.apache.fory.Fory; @@ -204,6 +205,34 @@ private FinalPostCtorBean(int value, String label) { } } + public static class PrivateNoArgParent { + static int noArgCalls; + private final String parentName; + + private PrivateNoArgParent() { + noArgCalls++; + parentName = "no-arg"; + } + + PrivateNoArgParent(String parentName) { + this.parentName = parentName; + } + + String parentName() { + return parentName; + } + } + + public static final class SerializablePrivateParentBean extends PrivateNoArgParent + implements Serializable { + private String childName; + + public SerializablePrivateParentBean(String parentName, String childName) { + super(parentName); + this.childName = childName; + } + } + @Test public void testFinalNoArgRestore() { FinalNoArgBean value = new FinalNoArgBean(7, "source", 9); @@ -280,6 +309,30 @@ public void testFinalPostCtorCodegen() { assertEquals(newValue.label, value.label); } + @Test + public void testNormalObjectSerializerBypassesObjectStreamParentRules() { + SerializablePrivateParentBean value = new SerializablePrivateParentBean("parent", "child"); + for (boolean codegen : new boolean[] {false, true}) { + PrivateNoArgParent.noArgCalls = 0; + Fory fory = + Fory.builder() + .withXlang(false) + .withRefTracking(true) + .withCodegen(codegen) + .requireClassRegistration(false) + .build(); + Serializer serializer = + fory.getTypeResolver().getTypeInfo(SerializablePrivateParentBean.class).getSerializer(); + Assert.assertFalse( + serializer instanceof ObjectStreamSerializer, serializer.getClass().getName()); + SerializablePrivateParentBean newValue = + (SerializablePrivateParentBean) fory.deserialize(fory.serialize(value)); + assertEquals(newValue.parentName(), value.parentName()); + assertEquals(newValue.childName, value.childName); + assertEquals(PrivateNoArgParent.noArgCalls, 0); + } + } + private static T roundTripWithSerializer(Fory fory, Serializer serializer, T value) { MemoryBuffer buffer = MemoryUtils.buffer(32); withWriteContext( diff --git a/java/fory-core/src/test/java/org/apache/fory/serializer/collection/AndroidCollectionFeatureTest.java b/java/fory-core/src/test/java/org/apache/fory/serializer/collection/AndroidCollectionFeatureTest.java index 0f6cda5edd..ac4f0d7eb6 100644 --- a/java/fory-core/src/test/java/org/apache/fory/serializer/collection/AndroidCollectionFeatureTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/serializer/collection/AndroidCollectionFeatureTest.java @@ -69,6 +69,8 @@ public void testAndroidCollectionFeaturePaths() throws Exception { String jvmEnumMapPayload = encode(jvmFory, newEnumMapValue()); String jvmEmptyEnumMapPayload = encode(jvmFory, new EnumMap<>(AndroidCollectionFeatureProbe.TestEnum.class)); + String jvmArrayQueuePayload = encode(jvmFory, newArrayBlockingQueueValue()); + String jvmLinkedQueuePayload = encode(jvmFory, newLinkedBlockingQueueValue()); String javaBin = System.getProperty("java.home") + File.separator + "bin" + File.separator + "java"; @@ -77,6 +79,8 @@ public void testAndroidCollectionFeaturePaths() throws Exception { if (!System.getProperty("java.specification.version").startsWith("1.")) { command.add("--add-opens"); command.add("java.base/java.util=ALL-UNNAMED"); + command.add("--add-opens"); + command.add("java.base/java.lang.invoke=ALL-UNNAMED"); } command.add("-cp"); command.add(System.getProperty("java.class.path")); @@ -85,6 +89,8 @@ public void testAndroidCollectionFeaturePaths() throws Exception { command.add(jvmSubListPayload); command.add(jvmEnumMapPayload); command.add(jvmEmptyEnumMapPayload); + command.add(jvmArrayQueuePayload); + command.add(jvmLinkedQueuePayload); ProcessBuilder processBuilder = new ProcessBuilder(command).redirectErrorStream(true); processBuilder.environment().put("FORY_ANDROID_ENABLED", "1"); Process process = processBuilder.start(); @@ -171,11 +177,25 @@ private static EnumMap newEnumMa return map; } + private static ArrayBlockingQueue newArrayBlockingQueueValue() { + ArrayBlockingQueue queue = new ArrayBlockingQueue<>(5); + queue.add("a"); + queue.add("b"); + return queue; + } + + private static LinkedBlockingQueue newLinkedBlockingQueueValue() { + LinkedBlockingQueue queue = new LinkedBlockingQueue<>(6); + queue.add("a"); + queue.add("b"); + return queue; + } + public static final class AndroidCollectionFeatureProbe { public static void main(String[] args) throws Exception { check( - args.length == 4, - "Expected JVM child-container, sublist, enum-map, and empty-enum-map payloads"); + args.length == 6, + "Expected JVM child-container, sublist, enum-map, empty-enum-map, and queue payloads"); System.setProperty("java.vm.name", "Dalvik"); System.setProperty("java.runtime.name", "Android Runtime"); check(AndroidSupport.IS_ANDROID, "AndroidSupport should detect Dalvik runtime"); @@ -195,7 +215,6 @@ public static void main(String[] args) throws Exception { verifySubList(fory); verifyArraysAsList(fory); verifySetFromMap(fory); - verifyQueues(fory); verifyEnumMap(fory); if (JdkVersion.MAJOR_VERSION >= 9) { verifyImmutableCollections(fory); @@ -204,6 +223,8 @@ public static void main(String[] args) throws Exception { verifyJvmSubListPayload(fory, Base64.getDecoder().decode(args[1])); verifyJvmEnumMapPayloads( fory, Base64.getDecoder().decode(args[2]), Base64.getDecoder().decode(args[3])); + verifyJvmQueuePayloads( + fory, Base64.getDecoder().decode(args[4]), Base64.getDecoder().decode(args[5])); writeAndroidFeaturePayloads(fory); } @@ -316,23 +337,24 @@ private static void verifySetFromMap(Fory fory) { checkEquals(copy, set, "SetFromMap copy"); } - private static void verifyQueues(Fory fory) { - ArrayBlockingQueue arrayQueue = new ArrayBlockingQueue<>(5); - arrayQueue.add("a"); - arrayQueue.add("b"); + private static void verifyJvmQueuePayloads( + Fory fory, byte[] jvmArrayQueuePayload, byte[] jvmLinkedQueuePayload) { + ArrayBlockingQueue arrayQueue = newArrayBlockingQueueValue(); ArrayBlockingQueue arrayRestored = - (ArrayBlockingQueue) roundTrip(fory, arrayQueue); + (ArrayBlockingQueue) fory.deserialize(jvmArrayQueuePayload); checkEquals( - new ArrayList<>(arrayRestored), new ArrayList<>(arrayQueue), "ArrayBlockingQueue"); + new ArrayList<>(arrayRestored), + new ArrayList<>(arrayQueue), + "JVM ArrayBlockingQueue payload"); checkEquals(arrayRestored.remainingCapacity(), 3, "ArrayBlockingQueue capacity"); - LinkedBlockingQueue linkedQueue = new LinkedBlockingQueue<>(6); - linkedQueue.add("a"); - linkedQueue.add("b"); + LinkedBlockingQueue linkedQueue = newLinkedBlockingQueueValue(); LinkedBlockingQueue linkedRestored = - (LinkedBlockingQueue) roundTrip(fory, linkedQueue); + (LinkedBlockingQueue) fory.deserialize(jvmLinkedQueuePayload); checkEquals( - new ArrayList<>(linkedRestored), new ArrayList<>(linkedQueue), "LinkedBlockingQueue"); + new ArrayList<>(linkedRestored), + new ArrayList<>(linkedQueue), + "JVM LinkedBlockingQueue payload"); checkEquals(linkedRestored.remainingCapacity(), 4, "LinkedBlockingQueue capacity"); } diff --git a/java/fory-core/src/test/java/org/apache/fory/serializer/collection/CollectionSerializersTest.java b/java/fory-core/src/test/java/org/apache/fory/serializer/collection/CollectionSerializersTest.java index 78996cbca4..2a97fbf202 100644 --- a/java/fory-core/src/test/java/org/apache/fory/serializer/collection/CollectionSerializersTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/serializer/collection/CollectionSerializersTest.java @@ -75,6 +75,7 @@ import org.apache.fory.exception.SerializationException; import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.memory.MemoryUtils; +import org.apache.fory.reflect.FieldAccessor; import org.apache.fory.reflect.TypeRef; import org.apache.fory.resolver.TypeResolver; import org.apache.fory.serializer.collection.CollectionSerializers.JDKCompatibleCollectionSerializer; @@ -990,6 +991,11 @@ public void testSetFromMap(Fory fory) { if (fory.getConfig().trackingRef()) { assertSame(struct2.collection, struct2.set); } + Map synchronizedMap = Collections.synchronizedMap(new HashMap<>()); + set = Collections.newSetFromMap(synchronizedMap); + set.add("a"); + set.add("b"); + serDeCheck(fory, set); } @Test @@ -1052,12 +1058,28 @@ private RefMarker(String value) { } @Test(dataProvider = "foryCopyConfig") - public void testSetFromMapCopy(Fory fory) { + public void testSetFromMapCopy(Fory fory) throws Exception { final Set set = Collections.newSetFromMap(Maps.newConcurrentMap()); set.add("a"); set.add("b"); set.add(Cyclic.create(true)); copyCheck(fory, set); + + final Map synchronizedMap = Collections.synchronizedMap(new HashMap<>()); + final Set synchronizedSet = Collections.newSetFromMap(synchronizedMap); + synchronizedSet.add("a"); + synchronizedSet.add("b"); + synchronizedSet.add(Cyclic.create(true)); + Set copied = fory.copy(synchronizedSet); + assertEquals(copied, synchronizedSet); + if (MemoryUtils.JDK_COLLECTION_FIELD_ACCESS) { + assertEquals(setFromMapBackingMap(copied).getClass(), synchronizedMap.getClass()); + } + } + + private static Map setFromMapBackingMap(Set set) throws Exception { + Field mapField = set.getClass().getDeclaredField("m"); + return (Map) FieldAccessor.createAccessor(mapField).getObject(set); } @Test(dataProvider = "javaFory") diff --git a/java/fory-core/src/test/java/org/apache/fory/serializer/collection/MapSerializersTest.java b/java/fory-core/src/test/java/org/apache/fory/serializer/collection/MapSerializersTest.java index bfd83c6264..595380bb06 100644 --- a/java/fory-core/src/test/java/org/apache/fory/serializer/collection/MapSerializersTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/serializer/collection/MapSerializersTest.java @@ -40,6 +40,7 @@ import java.util.EnumMap; import java.util.HashMap; import java.util.HashSet; +import java.util.IdentityHashMap; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -176,14 +177,15 @@ public void basicTestCaseWithMultiConfig( // testTreeMap TreeMap map = new TreeMap<>( - (s1, s2) -> { - int delta = s1.length() - s2.length(); - if (delta == 0) { - return s1.compareTo(s2); - } else { - return delta; - } - }); + (Comparator & Serializable) + (s1, s2) -> { + int delta = s1.length() - s2.length(); + if (delta == 0) { + return s1.compareTo(s2); + } else { + return delta; + } + }); map.put("str1", "1"); map.put("str2", "1"); assertEquals(map, serDe(fory, map)); @@ -379,14 +381,15 @@ public void testTreeMap() { .build(); TreeMap map = new TreeMap<>( - (s1, s2) -> { - int delta = s1.length() - s2.length(); - if (delta == 0) { - return s1.compareTo(s2); - } else { - return delta; - } - }); + (Comparator & Serializable) + (s1, s2) -> { + int delta = s1.length() - s2.length(); + if (delta == 0) { + return s1.compareTo(s2); + } else { + return delta; + } + }); map.put("str1", "1"); map.put("str2", "1"); assertEquals(map, serDe(fory, map)); @@ -398,14 +401,15 @@ public void testTreeMap() { public void testTreeMap(Fory fory) { TreeMap map = new TreeMap<>( - (s1, s2) -> { - int delta = s1.length() - s2.length(); - if (delta == 0) { - return s1.compareTo(s2); - } else { - return delta; - } - }); + (Comparator & Serializable) + (s1, s2) -> { + int delta = s1.length() - s2.length(); + if (delta == 0) { + return s1.compareTo(s2); + } else { + return delta; + } + }); map.put("str1", "1"); map.put("str2", "1"); copyCheck(fory, map); @@ -413,6 +417,44 @@ public void testTreeMap(Fory fory) { copyCheck(fory, beanForMap); } + @Test + public void testIdentityHashMapSerializer() { + Fory fory = + builder().withXlang(false).withRefTracking(true).requireClassRegistration(false).build(); + Assert.assertEquals( + fory.getTypeResolver().getSerializerClass(IdentityHashMap.class), + MapSerializers.IdentityHashMapSerializer.class); + Assert.assertTrue( + MapSerializers.JDKCompatibleMapSerializer.class.isAssignableFrom( + MapSerializers.IdentityHashMapSerializer.class)); + + IdentityHashMap map = newIdentityHashMap(); + IdentityHashMap restored = (IdentityHashMap) serDe(fory, map); + assertIdentityHashMap(restored); + + IdentityHashMap copied = fory.copy(map); + Assert.assertNotSame(copied, map); + assertIdentityHashMap(copied); + } + + private static IdentityHashMap newIdentityHashMap() { + IdentityHashMap map = new IdentityHashMap<>(); + map.put(new String("a"), "first"); + map.put(new String("a"), "second"); + return map; + } + + private static void assertIdentityHashMap(IdentityHashMap map) { + Assert.assertEquals(map.size(), 2); + int equalAKeys = 0; + for (Object key : map.keySet()) { + if ("a".equals(key)) { + equalAKeys++; + } + } + Assert.assertEquals(equalAKeys, 2); + } + @Test public void testXlangTreeMapCopy() { Fory fory = diff --git a/java/fory-latest-jdk-tests/src/test/java/org/apache/fory/integration_tests/ImmutableCollectionSerializersTest.java b/java/fory-latest-jdk-tests/src/test/java/org/apache/fory/integration_tests/ImmutableCollectionSerializersTest.java index c473dcaf7f..81eac1a0f7 100644 --- a/java/fory-latest-jdk-tests/src/test/java/org/apache/fory/integration_tests/ImmutableCollectionSerializersTest.java +++ b/java/fory-latest-jdk-tests/src/test/java/org/apache/fory/integration_tests/ImmutableCollectionSerializersTest.java @@ -29,8 +29,6 @@ import lombok.Data; import org.apache.fory.Fory; import org.apache.fory.ThreadSafeFory; -import org.apache.fory.exception.CopyException; -import org.apache.fory.exception.SerializationException; import org.apache.fory.platform.JdkVersion; import org.apache.fory.test.bean.CollectionFields; import org.apache.fory.test.bean.MapFields; @@ -102,17 +100,33 @@ public void testSetFromMapIdentityJdk25() { if (JdkVersion.MAJOR_VERSION < 25) { return; } - Fory fory = Fory.builder().withXlang(false).build(); - fory.register(IdentityHashMap.class); + Fory fory = Fory.builder().withXlang(false).requireClassRegistration(false).build(); Set set = Collections.newSetFromMap(new IdentityHashMap<>()); set.add(new String("a")); set.add(new String("a")); Assert.assertEquals(set.size(), 2); - SerializationException exception = - Assert.expectThrows(SerializationException.class, () -> fory.serialize(set)); - Assert.assertTrue(exception.getCause() instanceof UnsupportedOperationException); - CopyException copyException = Assert.expectThrows(CopyException.class, () -> fory.copy(set)); - Assert.assertTrue(copyException.getCause() instanceof UnsupportedOperationException); + + Set restored = (Set) fory.deserialize(fory.serialize(set)); + assertIdentitySet(restored); + + Set copied = fory.copy(set); + assertIdentitySet(copied); + } + + private static void assertIdentitySet(Set set) { + Assert.assertEquals(set.size(), 2); + Object first = null; + Object second = null; + for (Object value : set) { + if (first == null) { + first = value; + } else { + second = value; + } + } + Assert.assertEquals(first, "a"); + Assert.assertEquals(second, "a"); + Assert.assertNotSame(first, second); } @Data diff --git a/java/fory-testsuite/pom.xml b/java/fory-testsuite/pom.xml index 0ca60605ca..9f141ca140 100644 --- a/java/fory-testsuite/pom.xml +++ b/java/fory-testsuite/pom.xml @@ -168,7 +168,7 @@ pathsep="${path.separator}"/> --add-modules=jdk.incubator.vector - ${project.basedir}/../fory-core/target/jpms-classes/java16 + ${project.basedir}/../fory-core/target/multi-release-classes/16 From 8308f6ed2756e9ab94398f1e3e1ac4605fbb635d Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Wed, 3 Jun 2026 14:32:20 +0800 Subject: [PATCH 68/69] cleanup code --- .agents/languages/java.md | 13 ++- ci/run_ci.sh | 18 ++-- ci/tasks/java.py | 15 ++-- .../fory/graalvm/FeatureTestExample.java | 66 ++++++++++++++ java/fory-core/pom.xml | 2 +- .../fory/builder/BaseObjectCodecBuilder.java | 64 +++++++++++--- .../org/apache/fory/builder/CodecBuilder.java | 33 +++++-- .../fory/builder/ObjectCodecBuilder.java | 6 +- .../apache/fory/platform/GraalvmSupport.java | 22 +++++ .../fory/reflect/ObjectInstantiators.java | 51 ++++++++--- .../apache/fory/resolver/ClassResolver.java | 6 ++ .../fory/serializer/JavaSerializer.java | 4 +- .../apache/fory/serializer/Serializers.java | 27 ------ .../GuavaCollectionSerializers.java | 50 ++++++----- .../fory-core/native-image.properties | 2 + .../test/java/org/apache/fory/ForyTest.java | 16 ++-- .../fory/GuavaOptionalDependencyTest.java | 87 +++++++++++++++++++ .../fory/serializer/JavaSerializerTest.java | 59 ++++++++++++- 18 files changed, 424 insertions(+), 117 deletions(-) diff --git a/.agents/languages/java.md b/.agents/languages/java.md index 60bf283305..a68fd7bd97 100644 --- a/.agents/languages/java.md +++ b/.agents/languages/java.md @@ -90,10 +90,15 @@ Load this file when changing anything under `java/` or when Java drives a cross- JDK25+ ReflectionFactory path uses trusted-lookup access to `jdk.internal.reflect.ReflectionFactory` in `java.base` and must not require `--add-opens=java.base/jdk.internal.reflect=...`; the only JDK25+ platform open remains `java.base/java.lang.invoke=org.apache.fory.core`. GraalVM JDK25+ - native-image Serializable empty-instance construction is the narrow exception: direct - ReflectionFactory serialization constructors can produce `Object` there, so it uses a cached - `ObjectStreamClass` and a private `ObjectStreamClass.newInstance` MethodHandle from - `_JDKAccess._trustedLookup`. If the instantiator is retained in the native-image heap, the + native-image ordinary serializers may use an `ObjectStreamClass.newInstance` MethodHandle only + for the exact Serializable case where the serialization constructor class is `Object`; that + preserves normal empty-instance semantics because no user superclass constructor can run. For + Serializable classes whose first non-Serializable superclass is not `Object`, fail explicitly + instead of importing ObjectStream parent-constructor rules into normal Fory object creation. + ObjectStream-compatible serializers are the broader stream-specific path: direct ReflectionFactory + serialization constructors can produce `Object` there, so they use a cached `ObjectStreamClass` + and a private `ObjectStreamClass.newInstance` MethodHandle from `_JDKAccess._trustedLookup`. If + the instantiator is retained in the native-image heap, the MethodHandle owner may be initialized at build time but the per-type `ObjectStreamClass` descriptor must be cached only at image runtime. `ForyGraalVMFeature` must register the matching serialization constructor target class for each Serializable hierarchy member so runtime-lazy diff --git a/ci/run_ci.sh b/ci/run_ci.sh index 5798e9972e..316d5c30c0 100755 --- a/ci/run_ci.sh +++ b/ci/run_ci.sh @@ -87,7 +87,7 @@ graalvm_test() { java_major=$(echo "$java_version" | cut -d. -f1) fi if [[ "$java_major" -ge 25 ]]; then - export JDK_JAVA_OPTIONS="$(jdk25_plus_options "$java_major" "ALL-UNNAMED") $(jdk25_javac_options)" + export JDK_JAVA_OPTIONS="$(jdk25_runtime_options "ALL-UNNAMED") $(jdk25_javac_options)" else unset JDK_JAVA_OPTIONS fi @@ -108,8 +108,8 @@ jdk25_access_options() { printf " %s" "--add-opens=java.base/java.lang.invoke=${fory_open_targets}" } -jdk25_plus_options() { - local fory_targets="${2:-org.apache.fory.core}" +jdk25_runtime_options() { + local fory_targets="${1:-org.apache.fory.core}" printf "%s" "$(jdk25_access_options "$fory_targets")" } @@ -134,7 +134,7 @@ use_jdk() { if [[ "$jdk" =~ zulu([0-9]+) ]]; then local java_major="${BASH_REMATCH[1]}" if [[ "$java_major" -ge 25 ]]; then - export JDK_JAVA_OPTIONS="$(jdk25_plus_options "$java_major") $(jdk25_javac_options)" + export JDK_JAVA_OPTIONS="$(jdk25_runtime_options) $(jdk25_javac_options)" else unset JDK_JAVA_OPTIONS fi @@ -219,17 +219,17 @@ jdk17_plus_tests() { fi JDK_JAVA_OPTIONS="--add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED" if [[ "$java_major" -ge 25 ]]; then - JDK_JAVA_OPTIONS="$JDK_JAVA_OPTIONS $(jdk25_javac_options)" + JDK_JAVA_OPTIONS="$JDK_JAVA_OPTIONS $(jdk25_runtime_options "ALL-UNNAMED") $(jdk25_javac_options)" fi export JDK_JAVA_OPTIONS echo "Executing fory java tests" cd "$ROOT/java" set +e if [[ "$java_major" -ge 25 ]]; then - # JDK25+ must be tested from the packaged multi-release artifact. Raw - # reactor test classes bypass META-INF/versions/25 and exercise the - # JDK8-24 root implementation instead. - mvn -T10 --batch-mode --no-transfer-progress clean install -DskipTests + # The JDK25+ profile overlays Surefire's classpath with Java25 replacement + # classes while keeping the test run unnamed. Keep JPMS coverage below as a + # separate named-module check. + mvn -T10 --batch-mode --no-transfer-progress clean install else mvn -T10 --batch-mode --no-transfer-progress install fi diff --git a/ci/tasks/java.py b/ci/tasks/java.py index 5c8ee50d14..8e6ecc0826 100644 --- a/ci/tasks/java.py +++ b/ci/tasks/java.py @@ -84,7 +84,7 @@ def jdk25_access_options(fory_targets="org.apache.fory.core"): ] -def jdk25_plus_options(java_version, fory_targets="org.apache.fory.core"): +def jdk25_runtime_options(fory_targets="org.apache.fory.core"): return jdk25_access_options(fory_targets) @@ -107,7 +107,7 @@ def jdk25_javac_options(): def set_jdk_options(java_version): if int(java_version) >= 25: os.environ["JDK_JAVA_OPTIONS"] = " ".join( - jdk25_plus_options(java_version) + jdk25_javac_options() + jdk25_runtime_options() + jdk25_javac_options() ) else: os.environ.pop("JDK_JAVA_OPTIONS", None) @@ -252,17 +252,16 @@ def run_jdk17_plus(java_version="17"): "--add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED" ] if int(java_version) >= 25: + jdk_options.extend(jdk25_runtime_options("ALL-UNNAMED")) jdk_options.extend(jdk25_javac_options()) os.environ["JDK_JAVA_OPTIONS"] = " ".join(jdk_options) common.cd_project_subdir("java") if int(java_version) >= 25: - # JDK25+ must be tested from the packaged multi-release artifact. Raw - # reactor test classes bypass META-INF/versions/25 and exercise the - # JDK8-24 root implementation instead. - common.exec_cmd( - "mvn -T10 --batch-mode --no-transfer-progress clean install -DskipTests" - ) + # The JDK25+ profile overlays Surefire's classpath with Java25 replacement + # classes while keeping the test run unnamed. Keep JPMS coverage below as a + # separate named-module check. + common.exec_cmd("mvn -T10 --batch-mode --no-transfer-progress clean install") os.environ.pop("JDK_JAVA_OPTIONS", None) logging.info(f"Executing JDK{java_version} JPMS tests") common.cd_project_subdir("integration_tests/jpms_tests") diff --git a/integration_tests/graalvm_tests/src/main/java/org/apache/fory/graalvm/FeatureTestExample.java b/integration_tests/graalvm_tests/src/main/java/org/apache/fory/graalvm/FeatureTestExample.java index eef5992ecf..1c2d52c3d5 100644 --- a/integration_tests/graalvm_tests/src/main/java/org/apache/fory/graalvm/FeatureTestExample.java +++ b/integration_tests/graalvm_tests/src/main/java/org/apache/fory/graalvm/FeatureTestExample.java @@ -26,6 +26,7 @@ import org.apache.fory.Fory; import org.apache.fory.builder.Generated; import org.apache.fory.platform.GraalvmSupport; +import org.apache.fory.platform.JdkVersion; import org.apache.fory.util.Preconditions; public class FeatureTestExample { @@ -44,6 +45,34 @@ public String getValue() { } } + public static class PrivateNoArgParent { + static int noArgCalls; + private final String parentName; + + private PrivateNoArgParent() { + noArgCalls++; + parentName = "no-arg"; + } + + PrivateNoArgParent(String parentName) { + this.parentName = parentName; + } + + String parentName() { + return parentName; + } + } + + public static final class SerializablePrivateParentBean extends PrivateNoArgParent + implements Serializable { + private String childName; + + public SerializablePrivateParentBean(String parentName, String childName) { + super(parentName); + this.childName = childName; + } + } + public interface TestInterface { String getValue(); } @@ -80,6 +109,7 @@ private static Fory createFory() { .requireClassRegistration(true) .build(); fory.register(PrivateConstructorClass.class); + fory.register(SerializablePrivateParentBean.class); fory.register(TestInvocationHandler.class); GraalvmSupport.registerProxySupport(TestInterface.class); fory.ensureSerializersCompiled(); @@ -99,6 +129,8 @@ public static void main(String[] args) { Preconditions.checkArgument("test-value".equals(deserialized.getValue())); System.out.println("Private constructor class test passed"); + testNormalObjectCreation(); + // Test proxy serialization TestInterface proxy = (TestInterface) @@ -112,4 +144,38 @@ public static void main(String[] args) { System.out.println("FeatureTestExample succeed"); } + + private static void testNormalObjectCreation() { + SerializablePrivateParentBean original = + new SerializablePrivateParentBean("parent-value", "child-value"); + PrivateNoArgParent.noArgCalls = 0; + try { + SerializablePrivateParentBean deserialized = + (SerializablePrivateParentBean) fory.deserialize(fory.serialize(original)); + Preconditions.checkArgument("parent-value".equals(deserialized.parentName())); + Preconditions.checkArgument("child-value".equals(deserialized.childName)); + Preconditions.checkArgument(PrivateNoArgParent.noArgCalls == 0); + System.out.println("Normal object creation test passed"); + } catch (RuntimeException e) { + if (GraalvmSupport.IN_GRAALVM_NATIVE_IMAGE && JdkVersion.MAJOR_VERSION >= 25) { + Preconditions.checkArgument( + hasMessage(e, "would change ordinary Fory object-creation semantics")); + Preconditions.checkArgument(PrivateNoArgParent.noArgCalls == 0); + System.out.println("Normal object creation unsupported test passed"); + return; + } + throw e; + } + } + + private static boolean hasMessage(Throwable throwable, String message) { + while (throwable != null) { + String throwableMessage = throwable.getMessage(); + if (throwableMessage != null && throwableMessage.contains(message)) { + return true; + } + throwable = throwable.getCause(); + } + return false; + } } diff --git a/java/fory-core/pom.xml b/java/fory-core/pom.xml index 00dac76a44..1dd276218f 100644 --- a/java/fory-core/pom.xml +++ b/java/fory-core/pom.xml @@ -254,7 +254,7 @@ jdk25-multi-release - [25,] + [25,) diff --git a/java/fory-core/src/main/java/org/apache/fory/builder/BaseObjectCodecBuilder.java b/java/fory-core/src/main/java/org/apache/fory/builder/BaseObjectCodecBuilder.java index c2e670e795..ce5bdae3db 100644 --- a/java/fory-core/src/main/java/org/apache/fory/builder/BaseObjectCodecBuilder.java +++ b/java/fory-core/src/main/java/org/apache/fory/builder/BaseObjectCodecBuilder.java @@ -1763,6 +1763,45 @@ private TypeRef getMapChunkLocalType(TypeRef typeRef) { return TypeRef.of(CodeGenerator.getSourcePublicAccessibleParentClass(rawType)); } + private TypeRef mapChunkLocalType(TypeRef typeRef, boolean monomorphic) { + if (monomorphic || useCollectionSerialization(typeRef) || useMapSerialization(typeRef)) { + return getMapChunkLocalType(typeRef); + } + return OBJECT_TYPE; + } + + private Expression mapEntryRead(Expression entry, String methodName, TypeRef localType) { + Expression value = inlineInvoke(entry, methodName, OBJECT_TYPE); + if (localType.equals(OBJECT_TYPE)) { + return value; + } + return new Cast(value, localType, "mapEntryValue", true, false); + } + + private Expression mapEntryLocal( + Expression entry, String methodName, String namePrefix, TypeRef localType) { + if (localType.equals(OBJECT_TYPE)) { + return new Invoke(entry, methodName, namePrefix, OBJECT_TYPE); + } + return new Cast( + inlineInvoke(entry, methodName, OBJECT_TYPE), localType, namePrefix, false, false); + } + + private Expression writeMapChunkElement( + Expression value, + Expression buffer, + TypeRef typeRef, + Expression serializer, + boolean monomorphic) { + if (!monomorphic && value.type().equals(OBJECT_TYPE)) { + // Dynamic non-container chunks already wrote runtime type info. Keep the local as Object so + // raw maps with a value outside the declared generic type do not fail before that serializer + // gets the value. + return new Invoke(serializer, writeMethodName, writeContextRef(), value); + } + return serializeForNotNull(value, buffer, typeRef, serializer); + } + protected Expression writeChunk( Expression buffer, Expression entry, @@ -1774,16 +1813,16 @@ protected Expression writeChunk( boolean valueMonomorphic = isMonomorphic(valueType); Class keyTypeRawType = keyType.getRawType(); Class valueTypeRawType = valueType.getRawType(); - TypeRef keyLocalType = keyMonomorphic ? getMapChunkLocalType(keyType) : keyType; - TypeRef valueLocalType = valueMonomorphic ? getMapChunkLocalType(valueType) : valueType; + TypeRef keyLocalType = mapChunkLocalType(keyType, keyMonomorphic); + TypeRef valueLocalType = mapChunkLocalType(valueType, valueMonomorphic); Expression key = keyMonomorphic ? new Variable("key", keyLocalType) - : invoke(entry, "getKey", "key", keyType); + : mapEntryLocal(entry, "getKey", "key", keyLocalType); Expression value = valueMonomorphic ? new Variable("value", valueLocalType) - : invoke(entry, "getValue", "value", valueType); + : mapEntryLocal(entry, "getValue", "value", valueLocalType); Expression keyTypeExpr = keyMonomorphic ? getClassExpr(keyTypeRawType) @@ -1905,9 +1944,11 @@ protected Expression writeChunk( new While( Literal.ofBoolean(true), () -> { - Expression keyAssign = new Assign(key, invokeInline(entry, "getKey", keyLocalType)); + // Map.Entry is public but generated codecs may be package peers of non-public key or + // value types. Invoke as Object, then cast with this builder's source-access rules. + Expression keyAssign = new Assign(key, mapEntryRead(entry, "getKey", keyLocalType)); Expression valueAssign = - new Assign(value, invokeInline(entry, "getValue", valueLocalType)); + new Assign(value, mapEntryRead(entry, "getValue", valueLocalType)); Expression breakCondition; if (keyMonomorphic && valueMonomorphic) { breakCondition = or(eqNull(key), eqNull(value)); @@ -1931,7 +1972,8 @@ protected Expression writeChunk( neq(inlineInvoke(key, "getClass", CLASS_TYPE), keyTypeExpr), neq(inlineInvoke(value, "getClass", CLASS_TYPE), valueTypeExpr)); } - Expression writeKey = serializeForNotNull(key, buffer, keyType, keySerializer); + Expression writeKey = + writeMapChunkElement(key, buffer, keyType, keySerializer, keyMonomorphic); if (trackingKeyRef) { writeKey = new If( @@ -1946,7 +1988,7 @@ protected Expression writeChunk( writeKey); } Expression writeValue = - serializeForNotNull(value, buffer, valueType, valueSerializer); + writeMapChunkElement(value, buffer, valueType, valueSerializer, valueMonomorphic); if (trackingValueRef) { writeValue = new If( @@ -2277,9 +2319,11 @@ protected boolean hasCompatibleCollectionArrayRead(Descriptor descriptor) { return CompatibleSerializer.hasCompatibleCollectionArrayRead(typeResolver, descriptor); } - protected TypeRef compatibleReadTargetTypeRef(Descriptor descriptor) { + protected TypeRef compatibleLocalTypeRef(Descriptor descriptor) { + // Compatible collection/array metadata may keep the peer wire shape in TypeRef. Generated + // setters must cast to the local carrier that readCompatibleCollectionArrayField materializes. return descriptor.getField() == null - ? descriptor.getTypeRef() + ? TypeRef.of(descriptor.getRawType()) : TypeRef.of(descriptor.getField().getGenericType()); } diff --git a/java/fory-core/src/main/java/org/apache/fory/builder/CodecBuilder.java b/java/fory-core/src/main/java/org/apache/fory/builder/CodecBuilder.java index 46ff0aebda..68db4af0fc 100644 --- a/java/fory-core/src/main/java/org/apache/fory/builder/CodecBuilder.java +++ b/java/fory-core/src/main/java/org/apache/fory/builder/CodecBuilder.java @@ -36,6 +36,7 @@ import java.lang.invoke.MethodHandle; import java.lang.reflect.Field; +import java.lang.reflect.Method; import java.lang.reflect.Modifier; import java.util.HashMap; import java.util.Map; @@ -465,6 +466,8 @@ private Reference getFieldAccessor(Descriptor descriptor) { /** Returns an expression that set field value to bean. */ protected Expression setFieldValue(Expression bean, Descriptor d, Expression value) { String fieldName = d.getName(); + TypeRef memberType = memberTypeRef(d); + Class memberRawType = getRawType(memberType); if (value instanceof Inlineable) { ((Inlineable) value).inline(); } @@ -473,22 +476,22 @@ protected Expression setFieldValue(Expression bean, Descriptor d, Expression val } if (!d.isFinalField() && Modifier.isPublic(d.getModifiers()) - && sourcePublicAccessible(d.getRawType())) { - if (!d.getRawType().isAssignableFrom(value.type().getRawType())) { - value = tryInlineCast(value, d.getTypeRef()); + && sourcePublicAccessible(memberRawType)) { + if (!memberRawType.isAssignableFrom(value.type().getRawType())) { + value = tryInlineCast(value, memberType); } return new Expression.SetField(bean, fieldName, value); } else if (d.getWriteMethod() != null && Modifier.isPublic(d.getWriteMethod().getModifiers())) { - if (!d.getRawType().isAssignableFrom(value.type().getRawType())) { - value = tryInlineCast(value, d.getTypeRef()); + if (!memberRawType.isAssignableFrom(value.type().getRawType())) { + value = tryInlineCast(value, memberType); } return new Invoke(bean, d.getWriteMethod().getName(), value); } else { if (!d.isFinalField() && !Modifier.isPrivate(d.getModifiers())) { if (AccessorHelper.defineSetter(d.getField())) { Class accessorClass = AccessorHelper.getAccessorClass(d.getField()); - if (!d.getRawType().isAssignableFrom(value.type().getRawType())) { - value = tryInlineCast(value, d.getTypeRef()); + if (!memberRawType.isAssignableFrom(value.type().getRawType())) { + value = tryInlineCast(value, memberType); } return new StaticInvoke( accessorClass, d.getName(), PRIMITIVE_VOID_TYPE, false, bean, value); @@ -497,8 +500,8 @@ && sourcePublicAccessible(d.getRawType())) { if (d.getWriteMethod() != null && !Modifier.isPrivate(d.getWriteMethod().getModifiers())) { if (AccessorHelper.defineSetter(d.getWriteMethod())) { Class accessorClass = AccessorHelper.getAccessorClass(d.getWriteMethod()); - if (!d.getRawType().isAssignableFrom(value.type().getRawType())) { - value = tryInlineCast(value, d.getTypeRef()); + if (!memberRawType.isAssignableFrom(value.type().getRawType())) { + value = tryInlineCast(value, memberType); } return new StaticInvoke( accessorClass, d.getWriteMethod().getName(), PRIMITIVE_VOID_TYPE, false, bean, value); @@ -508,6 +511,18 @@ && sourcePublicAccessible(d.getRawType())) { } } + private TypeRef memberTypeRef(Descriptor descriptor) { + Field field = descriptor.getField(); + if (field != null) { + return TypeRef.of(field.getGenericType()); + } + Method method = descriptor.getWriteMethod(); + if (method != null) { + return TypeRef.of(method.getGenericParameterTypes()[0]); + } + return descriptor.getTypeRef(); + } + /** * Returns an expression that set field value to bean using reflection. */ diff --git a/java/fory-core/src/main/java/org/apache/fory/builder/ObjectCodecBuilder.java b/java/fory-core/src/main/java/org/apache/fory/builder/ObjectCodecBuilder.java index 41e5abb0c2..edb7eccb0e 100644 --- a/java/fory-core/src/main/java/org/apache/fory/builder/ObjectCodecBuilder.java +++ b/java/fory-core/src/main/java/org/apache/fory/builder/ObjectCodecBuilder.java @@ -906,9 +906,7 @@ protected Expression deserializeGroup( ExpressionVisitor.ExprHolder exprHolder = ExpressionVisitor.ExprHolder.of("bean", bean); walkPath.add(d.getDeclaringClass() + d.getName()); TypeRef castTypeRef = - hasCompatibleCollectionArrayRead(d) - ? compatibleReadTargetTypeRef(d) - : d.getTypeRef(); + hasCompatibleCollectionArrayRead(d) ? compatibleLocalTypeRef(d) : d.getTypeRef(); Expression action = deserializeField( buffer, @@ -948,7 +946,7 @@ protected Expression deserializeGroupForRecord( // use Reference to cut-off expr dependency. for (Descriptor d : group) { TypeRef castTypeRef = - hasCompatibleCollectionArrayRead(d) ? compatibleReadTargetTypeRef(d) : d.getTypeRef(); + hasCompatibleCollectionArrayRead(d) ? compatibleLocalTypeRef(d) : d.getTypeRef(); Expression value = deserializeField(buffer, d, expr -> expr); Expression action = setFieldValue(bean, d, tryInlineCast(value, castTypeRef)); groupExpressions.add(action); diff --git a/java/fory-core/src/main/java/org/apache/fory/platform/GraalvmSupport.java b/java/fory-core/src/main/java/org/apache/fory/platform/GraalvmSupport.java index aa119f2e60..4bb96a9704 100644 --- a/java/fory-core/src/main/java/org/apache/fory/platform/GraalvmSupport.java +++ b/java/fory-core/src/main/java/org/apache/fory/platform/GraalvmSupport.java @@ -105,6 +105,18 @@ public class GraalvmSupport { registerDefaultSerializerClass(TimeSerializers.TimeZoneSerializer.class); registerDefaultSerializerClass(BufferSerializers.ByteBufferSerializer.class); registerDefaultSerializerClass(MapSerializers.StringKeyMapSerializer.class); + registerDefaultSerializerClassIfPresent( + "org.apache.fory.serializer.collection.GuavaCollectionSerializers" + + "$ImmutableIntArraySerializer"); + registerDefaultSerializerClassIfPresent( + "org.apache.fory.serializer.collection.GuavaCollectionSerializers" + + "$ImmutableMapFormSerializer"); + registerDefaultSerializerClassIfPresent( + "org.apache.fory.serializer.collection.GuavaCollectionSerializers" + + "$ImmutableBiMapFormSerializer"); + registerDefaultSerializerClassIfPresent( + "org.apache.fory.serializer.collection.GuavaCollectionSerializers" + + "$HashBasedTableSerializer"); registerDefaultSerializerClass(ChildContainerSerializers.ChildArrayListSerializer.class); registerDefaultSerializerClass(ChildContainerSerializers.ChildCollectionSerializer.class); registerDefaultSerializerClass(ChildContainerSerializers.ChildMapSerializer.class); @@ -297,6 +309,16 @@ private static void registerDefaultSerializerClass(Class s DEFAULT_SERIALIZER_CLASSES.add(serializerClass); } + private static void registerDefaultSerializerClassIfPresent(String serializerClassName) { + try { + Class serializerClass = + Class.forName(serializerClassName, false, GraalvmSupport.class.getClassLoader()); + registerDefaultSerializerClass(serializerClass.asSubclass(Serializer.class)); + } catch (ClassNotFoundException | LinkageError e) { + // Guava is optional at runtime; only available Guava serializers should be registered. + } + } + public static ForyException throwNoArgCtrException(Class type) { throw new ForyException("Please provide a no-arg constructor for " + type); } diff --git a/java/fory-core/src/main/java/org/apache/fory/reflect/ObjectInstantiators.java b/java/fory-core/src/main/java/org/apache/fory/reflect/ObjectInstantiators.java index 4043d63692..b40f4d85ad 100644 --- a/java/fory-core/src/main/java/org/apache/fory/reflect/ObjectInstantiators.java +++ b/java/fory-core/src/main/java/org/apache/fory/reflect/ObjectInstantiators.java @@ -103,14 +103,16 @@ public static ObjectInstantiator createObjectInstantiator(Class type) if (noArgConstructor != null) { return new DeclaredNoArgCtrInstantiator<>(type); } else if (JdkVersion.MAJOR_VERSION >= 25) { - if (Serializable.class.isAssignableFrom(type)) { - return new ObjectStreamInstantiator<>(type); + if (Serializable.class.isAssignableFrom(type) + && serializationConstructorClass(type) == Object.class) { + return new GraalvmObjectInstantiator<>(type); } return new UnsupportedObjectInstantiator<>( type, "GraalVM native image on JDK25+ cannot create " + type - + " without an accessible no-arg constructor or Serializable metadata"); + + " without an accessible no-arg constructor because ObjectStream construction " + + "would change ordinary Fory object-creation semantics"); } else { return new UnsafeObjectInstantiator<>(type); } @@ -279,10 +281,10 @@ public T newInstanceWithArguments(Object... arguments) { } } - public static final class ObjectStreamInstantiator extends ObjectInstantiator { + private abstract static class ObjectStreamClassInstantiator extends ObjectInstantiator { private volatile ObjectStreamClass objectStreamClass; - public ObjectStreamInstantiator(Class type) { + private ObjectStreamClassInstantiator(Class type) { super(type); if (!GraalvmSupport.isGraalBuildTime()) { objectStreamClass = ObjectStreamClass.lookupAny(type); @@ -316,6 +318,25 @@ private ObjectStreamClass objectStreamClass() { } } + public static final class ObjectStreamInstantiator extends ObjectStreamClassInstantiator { + public ObjectStreamInstantiator(Class type) { + super(type); + } + } + + static final class GraalvmObjectInstantiator extends ObjectStreamClassInstantiator { + GraalvmObjectInstantiator(Class type) { + super(type); + if (!Serializable.class.isAssignableFrom(type) + || serializationConstructorClass(type) != Object.class) { + throw new ForyException( + "GraalVM JDK25+ ObjectStreamClass-backed object creation is only valid when " + + "the serialization constructor class is java.lang.Object for " + + type); + } + } + } + private static final class ObjectStreamAccess { private static final MethodHandle NEW_INSTANCE = newInstanceHandle(); @@ -403,15 +424,7 @@ private static Constructor findSerializationConstructor(Class type) if (!Serializable.class.isAssignableFrom(type)) { throw new ForyException("ObjectStream instantiation requires Serializable type " + type); } - Class current = type.getSuperclass(); - // Java ObjectStream reconstruction skips every Serializable class constructor and invokes - // only the first non-Serializable superclass no-arg constructor. - while (current != null && Serializable.class.isAssignableFrom(current)) { - current = current.getSuperclass(); - } - if (current == null) { - current = Object.class; - } + Class current = serializationConstructorClass(type); Constructor constructor = current.getDeclaredConstructor(); if (!validSerializationConstructor(type, current, constructor)) { throw new ForyException( @@ -450,6 +463,16 @@ public T newInstanceWithArguments(Object... arguments) { } } + private static Class serializationConstructorClass(Class type) { + Class current = type.getSuperclass(); + // Java ObjectStream reconstruction skips every Serializable class constructor and invokes only + // the first non-Serializable superclass no-arg constructor. + while (current != null && Serializable.class.isAssignableFrom(current)) { + current = current.getSuperclass(); + } + return current == null ? Object.class : current; + } + private static final class ReflectionFactoryAccess { private static final Object REFLECTION_FACTORY; private static final MethodHandle NEW_CONSTRUCTOR_FOR_SERIALIZATION; diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java index a23f2d20ab..d5b45fab05 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java @@ -1482,6 +1482,12 @@ public Class getSerializerClass(Class cls, boolean code if (serializerClass != null) { return serializerClass; } + if (config.registerGuavaTypes()) { + serializerClass = GuavaCollectionSerializers.getSerializerClass(cls); + if (serializerClass != null) { + return serializerClass; + } + } if (config.checkJdkClassSerializable()) { if (cls.getName().startsWith("java") && !(Serializable.class.isAssignableFrom(cls))) { throw new UnsupportedOperationException( diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/JavaSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/JavaSerializer.java index db3eb12c97..d0c2007f82 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/JavaSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/JavaSerializer.java @@ -120,6 +120,8 @@ public Object read(ReadContext readContext) { @Override public Object copy(CopyContext copyContext, Object value) { + // JavaSerializer copy must run the Java serialization lifecycle because readObject can rebuild + // transient state that object-field copy cannot infer. try { ByteArrayOutputStream bytes = new ByteArrayOutputStream(); try (ObjectOutputStream output = new ObjectOutputStream(bytes)) { @@ -127,7 +129,7 @@ public Object copy(CopyContext copyContext, Object value) { } try (ObjectInputStream input = new ClassLoaderObjectInputStream( - typeResolver.getClassLoader(), new ByteArrayInputStream(bytes.toByteArray()))) { + typeResolver, new ByteArrayInputStream(bytes.toByteArray()))) { return input.readObject(); } } catch (IOException | ClassNotFoundException e) { diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/Serializers.java b/java/fory-core/src/main/java/org/apache/fory/serializer/Serializers.java index 042e69e2d9..c0e0cf5b33 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/Serializers.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/Serializers.java @@ -30,7 +30,6 @@ import java.math.BigInteger; import java.net.URI; import java.nio.charset.Charset; -import java.util.Comparator; import java.util.Currency; import java.util.UUID; import java.util.concurrent.atomic.AtomicBoolean; @@ -48,7 +47,6 @@ import org.apache.fory.context.CopyContext; import org.apache.fory.context.ReadContext; import org.apache.fory.context.WriteContext; -import org.apache.fory.exception.ForyException; import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.memory.MemoryUtils; import org.apache.fory.meta.TypeDef; @@ -823,29 +821,6 @@ public Object read(ReadContext readContext) { } } - public static final class ReverseComparatorSerializer extends ImmutableSerializer - implements Shareable { - private static final byte ORIGINAL_REPLACE_RESOLVE_PAYLOAD = 0; - - public ReverseComparatorSerializer(Config config) { - super(config, (Class) Comparator.reverseOrder().getClass()); - } - - @Override - public void write(WriteContext writeContext, Comparator value) { - writeContext.getBuffer().writeByte(ORIGINAL_REPLACE_RESOLVE_PAYLOAD); - } - - @Override - public Comparator read(ReadContext readContext) { - byte payload = readContext.getBuffer().readByte(); - if (payload != ORIGINAL_REPLACE_RESOLVE_PAYLOAD) { - throw new ForyException("Unexpected reverse comparator payload flag " + payload); - } - return Comparator.reverseOrder(); - } - } - public static void registerDefaultSerializers(TypeResolver resolver) { Config config = resolver.getConfig(); resolver.registerInternalSerializer(Class.class, new ClassSerializer(config)); @@ -863,7 +838,5 @@ public static void registerDefaultSerializers(TypeResolver resolver) { resolver.registerInternalSerializer(Pattern.class, new RegexSerializer(config)); resolver.registerInternalSerializer(UUID.class, new UUIDSerializer(config)); resolver.registerInternalSerializer(Object.class, new EmptyObjectSerializer(config)); - resolver.registerInternalSerializer( - Comparator.reverseOrder().getClass(), new ReverseComparatorSerializer(config)); } } diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/GuavaCollectionSerializers.java b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/GuavaCollectionSerializers.java index 91e036088a..de0072decc 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/GuavaCollectionSerializers.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/GuavaCollectionSerializers.java @@ -62,16 +62,14 @@ public class GuavaCollectionSerializers { private static final String HASH_BASED_TABLE_CLASS_NAME = PKG + ".HashBasedTable"; private static final String IMMUTABLE_INT_ARRAY_CLASS_NAME = "com.google.common.primitives.ImmutableIntArray"; - private static final int NUM_RESERVED_TYPE_IDS = 17; + private static final int NUM_RESERVED_TYPE_IDS = 13; private static final boolean GUAVA_AVAILABLE = isClassAvailable(IMMUTABLE_BI_MAP_CLASS_NAME) && isClassAvailable(IMMUTABLE_LIST_CLASS_NAME) && isClassAvailable(IMMUTABLE_MAP_CLASS_NAME) && isClassAvailable(IMMUTABLE_SET_CLASS_NAME) && isClassAvailable(IMMUTABLE_SORTED_MAP_CLASS_NAME) - && isClassAvailable(IMMUTABLE_SORTED_SET_CLASS_NAME) - && isClassAvailable(HASH_BASED_TABLE_CLASS_NAME) - && isClassAvailable(IMMUTABLE_INT_ARRAY_CLASS_NAME); + && isClassAvailable(IMMUTABLE_SORTED_SET_CLASS_NAME); private interface MapEntryBuilder { void put(Object key, Object value); @@ -370,7 +368,7 @@ protected T xnewInstance(Map map) { } } - public static final class GuavaMapFormSerializer extends Serializer { + public abstract static class GuavaMapFormSerializer extends Serializer { private final Constructor constructor; private final Method readResolveMethod; private final boolean biMap; @@ -458,6 +456,18 @@ private static Method findReadResolve(Class cls) throws NoSuchMethodException } } + public static final class ImmutableMapFormSerializer extends GuavaMapFormSerializer { + public ImmutableMapFormSerializer(TypeResolver typeResolver, Class cls) { + super(typeResolver, cls, false); + } + } + + public static final class ImmutableBiMapFormSerializer extends GuavaMapFormSerializer { + public ImmutableBiMapFormSerializer(TypeResolver typeResolver, Class cls) { + super(typeResolver, cls, true); + } + } + public static final class ImmutableIntArraySerializer extends Serializer { public ImmutableIntArraySerializer(TypeResolver typeResolver, Class cls) { @@ -589,13 +599,8 @@ public T onMapRead(Map map) { // TODO guava serializers // guava/ArrayListMultimapSerializer - serializer for guava-libraries' ArrayListMultimap // guava/ArrayTableSerializer - serializer for guava-libraries' ArrayTable - // guava/HashBasedTableSerializer - serializer for guava-libraries' HashBasedTable // guava/HashMultimapSerializer -- serializer for guava-libraries' HashMultimap - // guava/ImmutableListSerializer - serializer for guava-libraries' ImmutableList - // guava/ImmutableSetSerializer - serializer for guava-libraries' ImmutableSet - // guava/ImmutableMapSerializer - serializer for guava-libraries' ImmutableMap // guava/ImmutableMultimapSerializer - serializer for guava-libraries' ImmutableMultimap - // guava/ImmutableSortedSetSerializer - serializer for guava-libraries' ImmutableSortedSet // guava/ImmutableTableSerializer - serializer for guava-libraries' ImmutableTable // guava/LinkedHashMultimapSerializer - serializer for guava-libraries' LinkedHashMultimap // guava/LinkedListMultimapSerializer - serializer for guava-libraries' LinkedListMultimap @@ -675,21 +680,20 @@ class GuavaEmptySortedMap {} cls = GuavaEmptySortedMap.class; resolver.registerInternalSerializer(cls, new ImmutableSortedMapSerializer(resolver, cls)); } - cls = ImmutableIntArray.class; - resolver.registerInternalSerializer(cls, new ImmutableIntArraySerializer(resolver, cls)); - cls = loadClass(IMMUTABLE_MAP_FORM_CLASS_NAME); - resolver.registerInternalSerializer(cls, new GuavaMapFormSerializer(resolver, cls, false)); - cls = loadClass(IMMUTABLE_BI_MAP_FORM_CLASS_NAME); - resolver.registerInternalSerializer(cls, new GuavaMapFormSerializer(resolver, cls, true)); - cls = HashBasedTable.class; - resolver.registerInternalSerializer(cls, new HashBasedTableSerializer(resolver, cls)); } - static Class loadClass(String className) { - try { - return Class.forName(className, false, GuavaCollectionSerializers.class.getClassLoader()); - } catch (ClassNotFoundException e) { - throw new RuntimeException(e); + public static Class getSerializerClass(Class cls) { + switch (cls.getName()) { + case IMMUTABLE_INT_ARRAY_CLASS_NAME: + return ImmutableIntArraySerializer.class; + case IMMUTABLE_MAP_FORM_CLASS_NAME: + return ImmutableMapFormSerializer.class; + case IMMUTABLE_BI_MAP_FORM_CLASS_NAME: + return ImmutableBiMapFormSerializer.class; + case HASH_BASED_TABLE_CLASS_NAME: + return HashBasedTableSerializer.class; + default: + return null; } } diff --git a/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties b/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties index 9fca2c8305..a4019d07d3 100644 --- a/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties +++ b/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties @@ -260,6 +260,8 @@ Args=--initialize-at-build-time=org.apache.fory.annotation.ForyField$Dynamic,\ org.apache.fory.reflect.TypeRef,\ org.apache.fory.reflect.ObjectInstantiators,\ org.apache.fory.reflect.ObjectInstantiators$DeclaredNoArgCtrInstantiator,\ + org.apache.fory.reflect.ObjectInstantiators$GraalvmObjectInstantiator,\ + org.apache.fory.reflect.ObjectInstantiators$ObjectStreamClassInstantiator,\ org.apache.fory.reflect.ObjectInstantiators$ObjectStreamInstantiator,\ org.apache.fory.reflect.ObjectInstantiators$ObjectStreamAccess,\ org.apache.fory.reflect.ObjectInstantiators$ParentNoArgCtrInstantiator,\ diff --git a/java/fory-core/src/test/java/org/apache/fory/ForyTest.java b/java/fory-core/src/test/java/org/apache/fory/ForyTest.java index 7928fdebb1..b070b64777 100644 --- a/java/fory-core/src/test/java/org/apache/fory/ForyTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/ForyTest.java @@ -71,8 +71,8 @@ import org.apache.fory.serializer.EnumSerializerTest; import org.apache.fory.serializer.ExceptionSerializers; import org.apache.fory.serializer.ObjectSerializer; +import org.apache.fory.serializer.ReplaceResolveSerializer; import org.apache.fory.serializer.Serializer; -import org.apache.fory.serializer.Serializers; import org.apache.fory.test.bean.BeanA; import org.apache.fory.test.bean.Struct; import org.apache.fory.type.Descriptor; @@ -99,13 +99,17 @@ public void typedDeserializeRejectsOutOfBandRootHeaderWithoutBuffers() { @Test public void testReverseComparatorSerializer() { Fory fory = Fory.builder().withXlang(false).requireClassRegistration(false).build(); - Comparator comparator = Comparator.reverseOrder(); + Comparator comparator = Comparator.reverseOrder(); Serializer serializer = fory.getTypeResolver().getTypeInfo(comparator.getClass()).getSerializer(); - assertTrue(serializer instanceof Serializers.ReverseComparatorSerializer); - Object roundTrip = fory.deserialize(fory.serialize(comparator)); - Assert.assertSame(roundTrip, Comparator.reverseOrder()); - Assert.assertSame(fory.copy(comparator), comparator); + assertTrue(serializer instanceof ReplaceResolveSerializer); + Comparator roundTrip = + (Comparator) fory.deserialize(fory.serialize(comparator)); + Assert.assertEquals(roundTrip.getClass(), comparator.getClass()); + Assert.assertEquals(roundTrip.compare(1, 2), comparator.compare(1, 2)); + Comparator copy = fory.copy(comparator); + Assert.assertEquals(copy.getClass(), comparator.getClass()); + Assert.assertEquals(copy.compare(2, 1), comparator.compare(2, 1)); } @Test diff --git a/java/fory-core/src/test/java/org/apache/fory/GuavaOptionalDependencyTest.java b/java/fory-core/src/test/java/org/apache/fory/GuavaOptionalDependencyTest.java index 476caef205..8eef899eea 100644 --- a/java/fory-core/src/test/java/org/apache/fory/GuavaOptionalDependencyTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/GuavaOptionalDependencyTest.java @@ -26,10 +26,16 @@ import java.io.File; import java.io.IOException; import java.io.InputStream; +import java.net.MalformedURLException; +import java.net.URL; +import java.net.URLClassLoader; import java.nio.charset.StandardCharsets; import java.util.Arrays; +import java.util.Set; import java.util.stream.Collectors; +import org.apache.fory.platform.GraalvmSupport; import org.apache.fory.resolver.ClassResolver; +import org.apache.fory.serializer.Serializer; import org.apache.fory.serializer.collection.GuavaCollectionSerializers; import org.testng.annotations.Test; @@ -39,6 +45,8 @@ public class GuavaOptionalDependencyTest { @Test public void testBuildWithoutGuavaAndReserveIds() throws Exception { assertTrue(GuavaCollectionSerializers.isGuavaAvailable()); + assertEquals(GuavaCollectionSerializers.getNumReservedTypeIds(), 13); + assertGraalvmGuavaSerializers(); RegistrationIds inProcessIds = currentProcessIds(); assertEquals( inProcessIds.enabledId - inProcessIds.disabledId, @@ -48,10 +56,30 @@ public void testBuildWithoutGuavaAndReserveIds() throws Exception { assertEquals(childIds.disabledId, inProcessIds.disabledId); } + @Test + public void testPartialGuavaReservesIds() throws Exception { + RegistrationIds inProcessIds = currentProcessIds(); + RegistrationIds childIds = runWithPartialGuava(); + assertEquals(childIds.enabledId, inProcessIds.enabledId); + assertEquals(childIds.disabledId, inProcessIds.disabledId); + } + private static RegistrationIds currentProcessIds() { return new RegistrationIds(registeredInternalId(true), registeredInternalId(false)); } + private static void assertGraalvmGuavaSerializers() { + Set> serializers = GraalvmSupport.getRegisteredSerializerClasses(); + assertTrue( + serializers.contains(GuavaCollectionSerializers.ImmutableIntArraySerializer.class)); + assertTrue( + serializers.contains(GuavaCollectionSerializers.ImmutableMapFormSerializer.class)); + assertTrue( + serializers.contains(GuavaCollectionSerializers.ImmutableBiMapFormSerializer.class)); + assertTrue( + serializers.contains(GuavaCollectionSerializers.HashBasedTableSerializer.class)); + } + private static int registeredInternalId(boolean registerGuavaTypes) { Fory fory = Fory.builder() @@ -76,6 +104,14 @@ private static RegistrationIds runWithoutGuava() throws Exception { return parseResult(output); } + private static RegistrationIds runWithPartialGuava() throws Exception { + try (PartialGuavaClassLoader loader = new PartialGuavaClassLoader(classPathUrls())) { + Class main = Class.forName(PartialGuavaMain.class.getName(), true, loader); + String output = (String) main.getMethod("run").invoke(null); + return parseResult(output); + } + } + private static RegistrationIds parseResult(String output) { for (String line : output.split("\\R")) { if (line.startsWith(RESULT_PREFIX)) { @@ -92,6 +128,20 @@ private static String removeGuavaFromClasspath(String classPath) { .collect(Collectors.joining(File.pathSeparator)); } + private static URL[] classPathUrls() { + return Arrays.stream(System.getProperty("java.class.path").split(File.pathSeparator)) + .map(GuavaOptionalDependencyTest::toUrl) + .toArray(URL[]::new); + } + + private static URL toUrl(String path) { + try { + return new File(path).toURI().toURL(); + } catch (MalformedURLException e) { + throw new IllegalArgumentException(e); + } + } + private static String readFully(InputStream inputStream) throws IOException { ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); byte[] buffer = new byte[1024]; @@ -114,6 +164,7 @@ private RegistrationIds(int enabledId, int disabledId) { public static final class NoGuavaMain { public static void main(String[] args) { + GraalvmSupport.getRegisteredSerializerClasses(); RegistrationIds ids = currentProcessIds(); Fory fory = Fory.builder() @@ -131,6 +182,42 @@ public static void main(String[] args) { } } + public static final class PartialGuavaMain { + public static String run() { + GraalvmSupport.getRegisteredSerializerClasses(); + RegistrationIds ids = currentProcessIds(); + Fory fory = + Fory.builder() + .withXlang(false) + .registerGuavaTypes(true) + .requireClassRegistration(false) + .suppressClassRegistrationWarnings(true) + .build(); + byte[] bytes = fory.serialize(com.google.common.collect.ImmutableList.of("fory")); + Object value = fory.deserialize(bytes); + if (!com.google.common.collect.ImmutableList.of("fory").equals(value)) { + throw new AssertionError("Unexpected round-trip value " + value); + } + return RESULT_PREFIX + ids.enabledId + "," + ids.disabledId; + } + } + + private static final class PartialGuavaClassLoader extends URLClassLoader { + private static final String FILTERED_CLASS = "com.google.common.primitives.ImmutableIntArray"; + + private PartialGuavaClassLoader(URL[] urls) { + super(urls, null); + } + + @Override + protected Class loadClass(String name, boolean resolve) throws ClassNotFoundException { + if (name.equals(FILTERED_CLASS)) { + throw new ClassNotFoundException(name); + } + return super.loadClass(name, resolve); + } + } + public static final class InternalSample {} public static final class SampleValue { diff --git a/java/fory-core/src/test/java/org/apache/fory/serializer/JavaSerializerTest.java b/java/fory-core/src/test/java/org/apache/fory/serializer/JavaSerializerTest.java index 9afcd8b1c0..6c76f12723 100644 --- a/java/fory-core/src/test/java/org/apache/fory/serializer/JavaSerializerTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/serializer/JavaSerializerTest.java @@ -31,6 +31,7 @@ import lombok.Data; import org.apache.fory.Fory; import org.apache.fory.ForyTestBase; +import org.apache.fory.exception.CopyException; import org.apache.fory.memory.BigEndian; import org.apache.fory.memory.MemoryBuffer; import org.testng.Assert; @@ -62,7 +63,32 @@ public static class JavaBox implements Serializable { } } - public static class NestedValue implements Serializable {} + public static class NestedValue implements Serializable { + String value; + + NestedValue() { + this("nested"); + } + + NestedValue(String value) { + this.value = value; + } + } + + public static class JavaCopyState implements Serializable { + String name; + transient int nameLength; + + JavaCopyState(String name) { + this.name = name; + this.nameLength = name.length(); + } + + private void readObject(java.io.ObjectInputStream stream) throws Exception { + stream.defaultReadObject(); + nameLength = name.length(); + } + } @Test public void testWriteObject() { @@ -98,6 +124,25 @@ public void testJdkSerializationCopy(Fory fory) throws MalformedURLException { copyCheck(fory, url); } + @Test + public void testCopyUsesJavaSerialization() { + Fory fory = + Fory.builder() + .withXlang(false) + .withRefCopy(true) + .requireClassRegistration(false) + .suppressClassRegistrationWarnings(true) + .build(); + fory.registerSerializer(JavaCopyState.class, JavaSerializer.class); + JavaCopyState state = new JavaCopyState("fory"); + state.nameLength = -1; + JavaCopyState copy = fory.copy(state); + + Assert.assertNotSame(copy, state); + Assert.assertEquals(copy.name, state.name); + Assert.assertEquals(copy.nameLength, 4); + } + @Test public void testJdkStreamChecksNestedClass() { Fory fory = Fory.builder().withXlang(false).build(); @@ -109,4 +154,16 @@ public void testJdkStreamChecksNestedClass() { Assert.assertThrows( InvalidClassException.class, () -> readSerializer(fory, serializer, buffer)); } + + @Test + public void testCopyChecksNestedClass() { + Fory fory = Fory.builder().withXlang(false).build(); + fory.register(JavaBox.class); + fory.registerSerializer(JavaBox.class, JavaSerializer.class); + + CopyException exception = + Assert.expectThrows(CopyException.class, () -> fory.copy(new JavaBox(new NestedValue()))); + Assert.assertTrue(exception.getCause() instanceof InvalidClassException); + Assert.assertTrue(exception.getCause().getMessage().contains(NestedValue.class.getName())); + } } From 3036e858001d2e0317577cabd246e2a29bf25e84 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Wed, 3 Jun 2026 15:22:22 +0800 Subject: [PATCH 69/69] fix(java): stabilize jdk25 copy and graalvm hooks --- .../apache/fory/platform/GraalvmSupport.java | 5 ++ .../serializer/ReplaceResolveSerializer.java | 63 ++++++++++++++----- .../collection/ChildContainerSerializers.java | 6 +- .../collection/CollectionSerializers.java | 8 ++- .../serializer/collection/ComparatorCopy.java | 43 +++++++++++++ .../GuavaCollectionSerializers.java | 4 +- .../serializer/collection/MapSerializers.java | 5 +- .../collection/SynchronizedSerializers.java | 5 +- .../collection/UnmodifiableSerializers.java | 5 +- .../fory/GuavaOptionalDependencyTest.java | 25 +++++--- .../test/java/org/apache/fory/TestUtils.java | 4 +- 11 files changed, 131 insertions(+), 42 deletions(-) create mode 100644 java/fory-core/src/main/java/org/apache/fory/serializer/collection/ComparatorCopy.java diff --git a/java/fory-core/src/main/java/org/apache/fory/platform/GraalvmSupport.java b/java/fory-core/src/main/java/org/apache/fory/platform/GraalvmSupport.java index 4bb96a9704..de036f9759 100644 --- a/java/fory-core/src/main/java/org/apache/fory/platform/GraalvmSupport.java +++ b/java/fory-core/src/main/java/org/apache/fory/platform/GraalvmSupport.java @@ -313,6 +313,11 @@ private static void registerDefaultSerializerClassIfPresent(String serializerCla try { Class serializerClass = Class.forName(serializerClassName, false, GraalvmSupport.class.getClassLoader()); + // Loading without initialization does not resolve member signatures. Optional serializers + // must prove their dependency types are present before entering the GraalVM metadata set. + serializerClass.getDeclaredConstructors(); + serializerClass.getDeclaredMethods(); + serializerClass.getDeclaredFields(); registerDefaultSerializerClass(serializerClass.asSubclass(Serializer.class)); } catch (ClassNotFoundException | LinkageError e) { // Guava is optional at runtime; only available Guava serializers should be registered. diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/ReplaceResolveSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/ReplaceResolveSerializer.java index eedd48052b..55d8e302c8 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/ReplaceResolveSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/ReplaceResolveSerializer.java @@ -21,6 +21,7 @@ import java.io.Externalizable; import java.io.Serializable; +import java.lang.invoke.MethodHandle; import java.lang.invoke.MethodHandles; import java.lang.reflect.Method; import java.util.HashMap; @@ -107,27 +108,16 @@ private ReplaceResolveInfo(Class cls) { : (readResolveMethod != null ? readResolveMethod.getDeclaringClass() : null); Function writeReplaceFunc = null, readResolveFunc = null; if (declaringClass != null) { - if (AndroidSupport.IS_ANDROID || GraalvmSupport.IN_GRAALVM_NATIVE_IMAGE) { + if (AndroidSupport.IS_ANDROID) { makeAccessible(writeReplaceMethod); makeAccessible(readResolveMethod); } else { MethodHandles.Lookup lookup = _JDKAccess._trustedLookup(declaringClass); - try { - if (writeReplaceMethod != null) { - writeReplaceFunc = - _JDKAccess.makeJDKFunction(lookup, lookup.unreflect(writeReplaceMethod)); - } - if (readResolveMethod != null) { - readResolveFunc = - _JDKAccess.makeJDKFunction(lookup, lookup.unreflect(readResolveMethod)); - } - } catch (Exception e) { - if (writeReplaceMethod != null && !writeReplaceMethod.isAccessible()) { - writeReplaceMethod.setAccessible(true); - } - if (readResolveMethod != null && !readResolveMethod.isAccessible()) { - readResolveMethod.setAccessible(true); - } + if (writeReplaceMethod != null) { + writeReplaceFunc = makeHookFunc(lookup, writeReplaceMethod); + } + if (readResolveMethod != null) { + readResolveFunc = makeHookFunc(lookup, readResolveMethod); } } } @@ -135,6 +125,28 @@ private ReplaceResolveInfo(Class cls) { this.readResolveFunc = readResolveFunc; } + private static Function makeHookFunc(MethodHandles.Lookup lookup, Method method) { + MethodHandle handle; + try { + handle = lookup.unreflect(method); + } catch (IllegalAccessException e) { + throw new ForyException( + "Failed to access Java replacement hook " + + method + + ". " + + _JDKAccess.jdk25AccessMessage(), + e); + } + try { + return _JDKAccess.makeJDKFunction(lookup, handle); + } catch (Throwable e) { + if (GraalvmSupport.IN_GRAALVM_NATIVE_IMAGE) { + return new MethodHandleFunction(handle); + } + throw ExceptionUtils.throwException(e); + } + } + private static void makeAccessible(Method method) { if (method == null) { return; @@ -146,6 +158,23 @@ private static void makeAccessible(Method method) { } } + private static final class MethodHandleFunction implements Function { + private final MethodHandle handle; + + private MethodHandleFunction(MethodHandle handle) { + this.handle = handle; + } + + @Override + public Object apply(Object value) { + try { + return handle.invoke(value); + } catch (Throwable e) { + throw ExceptionUtils.throwException(e); + } + } + } + Object writeReplace(Object o) { if (writeReplaceFunc != null) { return writeReplaceFunc.apply(o); diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/ChildContainerSerializers.java b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/ChildContainerSerializers.java index b3daa780a3..f7840349ef 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/ChildContainerSerializers.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/ChildContainerSerializers.java @@ -263,7 +263,7 @@ public T newCollection(ReadContext readContext) { public Collection newCollection(CopyContext copyContext, Collection originCollection) { T newCollection = subclassFactory.newCollection( - copyContext.copyObject(((SortedSet) originCollection).comparator())); + ComparatorCopy.copy(copyContext, ((SortedSet) originCollection).comparator())); copyChildFields(copyContext, originCollection, newCollection); return newCollection; } @@ -309,7 +309,7 @@ public T newCollection(ReadContext readContext) { public Collection newCollection(CopyContext copyContext, Collection originCollection) { T newCollection = subclassFactory.newCollection( - copyContext.copyObject(((PriorityQueue) originCollection).comparator()), + ComparatorCopy.copy(copyContext, ((PriorityQueue) originCollection).comparator()), originCollection.size()); copyChildFields(copyContext, originCollection, newCollection); return newCollection; @@ -417,7 +417,7 @@ public Map newMap(ReadContext readContext) { public Map newMap(CopyContext copyContext, Map originMap) { T newMap = subclassFactory.newMap( - copyContext.copyObject(((SortedMap) originMap).comparator())); + ComparatorCopy.copy(copyContext, ((SortedMap) originMap).comparator())); copyChildFields(copyContext, originMap, newMap); return newMap; } diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/CollectionSerializers.java b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/CollectionSerializers.java index e87bf27298..d820331c6f 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/CollectionSerializers.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/CollectionSerializers.java @@ -305,7 +305,8 @@ public T newCollection(ReadContext readContext) { @Override public Collection newCollection(CopyContext copyContext, Collection originCollection) { Collection collection; - Comparator comparator = copyContext.copyObject(((SortedSet) originCollection).comparator()); + Comparator comparator = + ComparatorCopy.copy(copyContext, ((SortedSet) originCollection).comparator()); if (Objects.equals(type, TreeSet.class)) { collection = new TreeSet(comparator); } else { @@ -570,7 +571,7 @@ public ConcurrentSkipListSet newCollection(ReadContext readContext) { @Override public Collection newCollection(CopyContext copyContext, Collection originCollection) { Comparator comparator = - copyContext.copyObject(((ConcurrentSkipListSet) originCollection).comparator()); + ComparatorCopy.copy(copyContext, ((ConcurrentSkipListSet) originCollection).comparator()); return new ConcurrentSkipListSet(comparator); } } @@ -869,7 +870,8 @@ public Collection onCollectionWrite(WriteContext writeContext, PriorityQueue val @Override public Collection newCollection(CopyContext copyContext, Collection collection) { return new PriorityQueue( - collection.size(), copyContext.copyObject(((PriorityQueue) collection).comparator())); + collection.size(), + ComparatorCopy.copy(copyContext, ((PriorityQueue) collection).comparator())); } @Override diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/ComparatorCopy.java b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/ComparatorCopy.java new file mode 100644 index 0000000000..23fc54bcf0 --- /dev/null +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/ComparatorCopy.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.serializer.collection; + +import java.util.Comparator; +import org.apache.fory.context.CopyContext; + +final class ComparatorCopy { + private static final Class NATURAL_ORDER_CLASS = Comparator.naturalOrder().getClass(); + private static final Class REVERSE_ORDER_CLASS = Comparator.reverseOrder().getClass(); + + private ComparatorCopy() {} + + static Comparator copy(CopyContext copyContext, Comparator comparator) { + if (comparator == null || isJdkSingleton(comparator)) { + // Xlang copy is JVM-local; immutable JDK comparator singletons do not need a wire type. + return comparator; + } + return copyContext.copyObject(comparator); + } + + private static boolean isJdkSingleton(Comparator comparator) { + Class type = comparator.getClass(); + return type == NATURAL_ORDER_CLASS || type == REVERSE_ORDER_CLASS; + } +} diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/GuavaCollectionSerializers.java b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/GuavaCollectionSerializers.java index de0072decc..ac4e801104 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/GuavaCollectionSerializers.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/GuavaCollectionSerializers.java @@ -218,7 +218,7 @@ public T onCollectionRead(Collection collection) { @Override public T copy(CopyContext copyContext, T originCollection) { - Comparator comparator = copyContext.copyObject(originCollection.comparator()); + Comparator comparator = ComparatorCopy.copy(copyContext, originCollection.comparator()); Object[] elements = new Object[originCollection.size()]; copyElements(copyContext, originCollection, elements); return (T) new ImmutableSortedSet.Builder<>(comparator).add(elements).build(); @@ -576,7 +576,7 @@ public Map newMap(ReadContext readContext) { @Override public T copy(CopyContext copyContext, T originMap) { - Comparator comparator = copyContext.copyObject(originMap.comparator()); + Comparator comparator = ComparatorCopy.copy(copyContext, originMap.comparator()); ImmutableSortedMap.Builder builder = new ImmutableSortedMap.Builder(comparator); copyEntries(typeResolver, mapTypeCache(), copyContext, originMap, builder::put); return (T) builder.build(); diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/MapSerializers.java b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/MapSerializers.java index dd1b754e3e..9b3825495a 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/MapSerializers.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/MapSerializers.java @@ -222,7 +222,8 @@ public Map newMap(ReadContext readContext) { @Override public Map newMap(CopyContext copyContext, Map originMap) { - Comparator comparator = copyContext.copyObject(((SortedMap) originMap).comparator()); + Comparator comparator = + ComparatorCopy.copy(copyContext, ((SortedMap) originMap).comparator()); Map map; if (type == TreeMap.class) { map = new TreeMap(comparator); @@ -369,7 +370,7 @@ public ConcurrentSkipListMap newMap(ReadContext readContext) { @Override public Map newMap(CopyContext copyContext, Map originMap) { Comparator comparator = - copyContext.copyObject(((ConcurrentSkipListMap) originMap).comparator()); + ComparatorCopy.copy(copyContext, ((ConcurrentSkipListMap) originMap).comparator()); return new ConcurrentSkipListMap(comparator); } } diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/SynchronizedSerializers.java b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/SynchronizedSerializers.java index a3e7893a6b..da66076745 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/SynchronizedSerializers.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/SynchronizedSerializers.java @@ -127,7 +127,7 @@ public Collection copy(CopyContext copyContext, Collection object) { synchronized (object) { Collection mutableSource; if (object instanceof SortedSet) { - Object comparator = copyContext.copyObject(((SortedSet) object).comparator()); + Object comparator = ComparatorCopy.copy(copyContext, ((SortedSet) object).comparator()); mutableSource = new TreeSet((java.util.Comparator) comparator); } else if (object instanceof Set) { mutableSource = new HashSet(object.size()); @@ -185,7 +185,8 @@ public Map copy(CopyContext copyContext, Map originMap) { synchronized (originMap) { Map mutableSource; if (originMap instanceof SortedMap) { - Object comparator = copyContext.copyObject(((SortedMap) originMap).comparator()); + Object comparator = + ComparatorCopy.copy(copyContext, ((SortedMap) originMap).comparator()); mutableSource = new TreeMap((java.util.Comparator) comparator); } else { mutableSource = new HashMap(originMap.size()); diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/UnmodifiableSerializers.java b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/UnmodifiableSerializers.java index a024391d46..ec96be1999 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/UnmodifiableSerializers.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/UnmodifiableSerializers.java @@ -119,7 +119,7 @@ public Collection copy(CopyContext copyContext, Collection object) { if (!MemoryUtils.JDK_COLLECTION_FIELD_ACCESS) { Collection mutableSource; if (object instanceof SortedSet) { - Object comparator = copyContext.copyObject(((SortedSet) object).comparator()); + Object comparator = ComparatorCopy.copy(copyContext, ((SortedSet) object).comparator()); mutableSource = new TreeSet((java.util.Comparator) comparator); } else if (object instanceof Set) { mutableSource = new HashSet(object.size()); @@ -168,7 +168,8 @@ public Map copy(CopyContext copyContext, Map originMap) { if (!MemoryUtils.JDK_COLLECTION_FIELD_ACCESS) { Map mutableSource; if (originMap instanceof SortedMap) { - Object comparator = copyContext.copyObject(((SortedMap) originMap).comparator()); + Object comparator = + ComparatorCopy.copy(copyContext, ((SortedMap) originMap).comparator()); mutableSource = new TreeMap((java.util.Comparator) comparator); } else { mutableSource = new HashMap(originMap.size()); diff --git a/java/fory-core/src/test/java/org/apache/fory/GuavaOptionalDependencyTest.java b/java/fory-core/src/test/java/org/apache/fory/GuavaOptionalDependencyTest.java index 8eef899eea..1e2f624f6e 100644 --- a/java/fory-core/src/test/java/org/apache/fory/GuavaOptionalDependencyTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/GuavaOptionalDependencyTest.java @@ -70,14 +70,10 @@ private static RegistrationIds currentProcessIds() { private static void assertGraalvmGuavaSerializers() { Set> serializers = GraalvmSupport.getRegisteredSerializerClasses(); - assertTrue( - serializers.contains(GuavaCollectionSerializers.ImmutableIntArraySerializer.class)); - assertTrue( - serializers.contains(GuavaCollectionSerializers.ImmutableMapFormSerializer.class)); - assertTrue( - serializers.contains(GuavaCollectionSerializers.ImmutableBiMapFormSerializer.class)); - assertTrue( - serializers.contains(GuavaCollectionSerializers.HashBasedTableSerializer.class)); + assertTrue(serializers.contains(GuavaCollectionSerializers.ImmutableIntArraySerializer.class)); + assertTrue(serializers.contains(GuavaCollectionSerializers.ImmutableMapFormSerializer.class)); + assertTrue(serializers.contains(GuavaCollectionSerializers.ImmutableBiMapFormSerializer.class)); + assertTrue(serializers.contains(GuavaCollectionSerializers.HashBasedTableSerializer.class)); } private static int registeredInternalId(boolean registerGuavaTypes) { @@ -164,7 +160,7 @@ private RegistrationIds(int enabledId, int disabledId) { public static final class NoGuavaMain { public static void main(String[] args) { - GraalvmSupport.getRegisteredSerializerClasses(); + assertSerializerMetadataLinked(); RegistrationIds ids = currentProcessIds(); Fory fory = Fory.builder() @@ -184,7 +180,7 @@ public static void main(String[] args) { public static final class PartialGuavaMain { public static String run() { - GraalvmSupport.getRegisteredSerializerClasses(); + assertSerializerMetadataLinked(); RegistrationIds ids = currentProcessIds(); Fory fory = Fory.builder() @@ -202,6 +198,15 @@ public static String run() { } } + private static void assertSerializerMetadataLinked() { + for (Class serializerClass : + GraalvmSupport.getRegisteredSerializerClasses()) { + serializerClass.getDeclaredConstructors(); + serializerClass.getDeclaredMethods(); + serializerClass.getDeclaredFields(); + } + } + private static final class PartialGuavaClassLoader extends URLClassLoader { private static final String FILTERED_CLASS = "com.google.common.primitives.ImmutableIntArray"; diff --git a/java/fory-core/src/test/java/org/apache/fory/TestUtils.java b/java/fory-core/src/test/java/org/apache/fory/TestUtils.java index 48cef098dc..81cfec51a8 100644 --- a/java/fory-core/src/test/java/org/apache/fory/TestUtils.java +++ b/java/fory-core/src/test/java/org/apache/fory/TestUtils.java @@ -81,7 +81,9 @@ public static List javaCommand( private static List forkJvmArgs() { List args = new ArrayList<>(); if (JdkVersion.MAJOR_VERSION >= 25) { - args.add("--add-opens=java.base/java.lang.invoke=org.apache.fory.core"); + // TestUtils.javaCommand launches probes with -cp, so Fory code is in the unnamed module. + // Named-module coverage lives in integration_tests/jpms_tests. + args.add("--add-opens=java.base/java.lang.invoke=ALL-UNNAMED"); if (hasInputArg("--sun-misc-unsafe-memory-access=deny")) { args.add("--sun-misc-unsafe-memory-access=deny"); }