diff --git a/java/fory-core/src/main/java/org/apache/fory/codegen/Expression.java b/java/fory-core/src/main/java/org/apache/fory/codegen/Expression.java index 185fb70c07..ce43201dd3 100644 --- a/java/fory-core/src/main/java/org/apache/fory/codegen/Expression.java +++ b/java/fory-core/src/main/java/org/apache/fory/codegen/Expression.java @@ -1597,7 +1597,7 @@ public String toString() { } } - class NewArray extends AbstractExpression { + class NewArray extends Inlineable { private TypeRef type; private Expression[] elements; @@ -1662,59 +1662,35 @@ public ExprCode doGenCode(CodegenContext ctx) { Class rawType = getRawType(type); String arrayType = getArrayType(rawType); String value = ctx.newName("arr"); + String arrayValue; if (dims != null) { // multi-dimension array ExprCode dimsExprCode = dims.genCode(ctx); if (StringUtils.isNotBlank(dimsExprCode.code())) { codeBuilder.append(dimsExprCode.code()).append('\n'); } - // "${arrType} ${value} = new ${elementType}[$?][$?]... - codeBuilder - .append(arrayType) - .append(' ') - .append(value) - .append(" = new ") - .append(ctx.type(elemType)); + // "new ${elementType}[$?][$?]..." + StringBuilder arrayValueBuilder = new StringBuilder("new ").append(ctx.type(elemType)); for (int i = 0; i < numDimensions; i++) { // dims is dimensions array, which store size of per dim. String idim = StringUtils.format("${dims}[${i}]", "dims", dimsExprCode.value(), "i", i); - codeBuilder.append('[').append(idim).append("]"); + arrayValueBuilder.append('[').append(idim).append("]"); } - codeBuilder.append(';'); + arrayValue = arrayValueBuilder.toString(); } else if (dim != null) { ExprCode dimExprCode = dim.genCode(ctx); if (StringUtils.isNotBlank(dimExprCode.code())) { codeBuilder.append(dimExprCode.code()).append('\n'); } + StringBuilder arrayValueBuilder = new StringBuilder("new ").append(ctx.type(elemType)); + arrayValueBuilder.append('[').append(dimExprCode.value()).append(']'); if (numDimensions > 1) { // multi-dimension array - // "${arrType} ${value} = new ${elementType}[$?][][][]... - codeBuilder - .append(arrayType) - .append(' ') - .append(value) - .append(" = new ") - .append(ctx.type(elemType)); - codeBuilder.append('[').append(dimExprCode.value()).append(']'); for (int i = 1; i < numDimensions; i++) { - codeBuilder.append('[').append("]"); + arrayValueBuilder.append('[').append("]"); } - codeBuilder.append(';'); - } else { - // one-dimension array - String code = - StringUtils.format( - "${type} ${value} = new ${elemType}[${dim}];", - "type", - arrayType, - "elemType", - ctx.type(elemType), - "value", - value, - "dim", - dimExprCode.value()); - codeBuilder.append(code); } + arrayValue = arrayValueBuilder.toString(); } else { // create array with init value int len = elements.length; @@ -1733,18 +1709,22 @@ public ExprCode doGenCode(CodegenContext ctx) { } } - String code = + arrayValue = StringUtils.format( - "${type} ${value} = new ${type} {${args}};", - "type", - arrayType, - "value", - value, - "args", - argsBuilder.toString()); - codeBuilder.append(code); + "new ${type} {${args}}", "type", arrayType, "args", argsBuilder.toString()); } + if (inlineCall) { + String code = StringUtils.isBlank(codeBuilder) ? null : codeBuilder.toString(); + return new ExprCode(code, null, Code.variable(rawType, arrayValue)); + } + codeBuilder + .append(arrayType) + .append(' ') + .append(value) + .append(" = ") + .append(arrayValue) + .append(';'); return new ExprCode(codeBuilder.toString(), null, Code.variable(rawType, value)); } } diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/StringSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/StringSerializer.java index 2ad935101f..537ef69876 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/StringSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/StringSerializer.java @@ -442,6 +442,38 @@ public static byte[] getStringBytes(String value) { return (byte[]) getStringValue(value); } + @Internal + public static void copyStringCharsToBytes(String value, byte[] target) { + int length = value.length(); + if (length > (Integer.MAX_VALUE >>> 1)) { + throw new IllegalArgumentException("String is too large"); + } + int numBytes = length << 1; + if (target.length < numBytes) { + throw new IllegalArgumentException("Target byte array is too small"); + } + if (STRING_VALUE_FIELD_IS_CHARS) { + char[] chars = (char[]) getStringValue(value); + int offset = STRING_HAS_COUNT_OFFSET ? getStringOffset(value) : 0; + PlatformStringUtils.copyCharsToBytes(chars, offset, target, 0, numBytes); + return; + } + int byteIndex = 0; + if (NativeByteOrder.IS_LITTLE_ENDIAN) { + for (int i = 0; i < length; i++) { + char c = value.charAt(i); + target[byteIndex++] = (byte) c; + target[byteIndex++] = (byte) (c >>> 8); + } + } else { + for (int i = 0; i < length; i++) { + char c = value.charAt(i); + target[byteIndex++] = (byte) (c >>> 8); + target[byteIndex++] = (byte) c; + } + } + } + @Internal public static byte getStringCoder(String value) { return PlatformStringUtils.getStringCoder(value); @@ -621,22 +653,26 @@ public byte[] readBytesUTF8(MemoryBuffer buffer, int numBytes) { } private byte[] readBytesUTF8PerfOptimized(MemoryBuffer buffer, int numBytes) { - int udf8Bytes = buffer.readInt32(); - checkStringSize(udf8Bytes); + int utf8Bytes = buffer.readInt32(); + checkUtf8DecodedBytes(numBytes, utf8Bytes); // noinspection Duplicates - buffer.checkReadableBytes(udf8Bytes); + buffer.checkReadableBytes(utf8Bytes); byte[] bytes = new byte[numBytes]; byte[] srcArray = buffer.getHeapMemory(); if (srcArray != null) { int srcIndex = buffer._unsafeHeapReaderIndex(); - int readLen = StringEncodingUtils.convertUTF8ToUTF16(srcArray, srcIndex, udf8Bytes, bytes); - assert readLen == numBytes : "Decode UTF8 to UTF16 failed"; - buffer._increaseReaderIndexUnsafe(udf8Bytes); + int readLen = StringEncodingUtils.convertUTF8ToUTF16(srcArray, srcIndex, utf8Bytes, bytes); + if (readLen != numBytes) { + throwUtf8DecodedSizeMismatch(numBytes, readLen); + } + buffer._increaseReaderIndexUnsafe(utf8Bytes); } else { - byte[] tmpArray = getByteArray(udf8Bytes); - buffer.readBytes(tmpArray, 0, udf8Bytes); - int readLen = StringEncodingUtils.convertUTF8ToUTF16(tmpArray, 0, udf8Bytes, bytes); - assert readLen == numBytes : "Decode UTF8 to UTF16 failed"; + byte[] tmpArray = getByteArray(utf8Bytes); + buffer.readBytes(tmpArray, 0, utf8Bytes); + int readLen = StringEncodingUtils.convertUTF8ToUTF16(tmpArray, 0, utf8Bytes, bytes); + if (readLen != numBytes) { + throwUtf8DecodedSizeMismatch(numBytes, readLen); + } } return bytes; } @@ -688,23 +724,27 @@ public String readCharsUTF8(MemoryBuffer buffer, int numBytes) { public String readCharsUTF8PerfOptimized(MemoryBuffer buffer, int numBytes) { checkStringSize(numBytes); + int utf8Bytes = buffer.readInt32(); + checkUtf8DecodedBytes(numBytes, utf8Bytes); int udf16Chars = numBytes >> 1; - int udf8Bytes = buffer.readInt32(); - checkStringSize(udf8Bytes); // noinspection Duplicates - buffer.checkReadableBytes(udf8Bytes); + buffer.checkReadableBytes(utf8Bytes); char[] chars = new char[udf16Chars]; byte[] srcArray = buffer.getHeapMemory(); if (srcArray != null) { int srcIndex = buffer._unsafeHeapReaderIndex(); - int readLen = StringEncodingUtils.convertUTF8ToUTF16(srcArray, srcIndex, udf8Bytes, chars); - assert readLen == udf16Chars : "Decode UTF8 to UTF16 failed"; - buffer._increaseReaderIndexUnsafe(udf8Bytes); + int readLen = StringEncodingUtils.convertUTF8ToUTF16(srcArray, srcIndex, utf8Bytes, chars); + if (readLen != udf16Chars) { + throwUtf8DecodedSizeMismatch(udf16Chars, readLen); + } + buffer._increaseReaderIndexUnsafe(utf8Bytes); } else { - byte[] tmpArray = getByteArray(udf8Bytes); - buffer.readBytes(tmpArray, 0, udf8Bytes); - int readLen = StringEncodingUtils.convertUTF8ToUTF16(tmpArray, 0, udf8Bytes, chars); - assert readLen == udf16Chars : "Decode UTF8 to UTF16 failed"; + byte[] tmpArray = getByteArray(utf8Bytes); + buffer.readBytes(tmpArray, 0, utf8Bytes); + int readLen = StringEncodingUtils.convertUTF8ToUTF16(tmpArray, 0, utf8Bytes, chars); + if (readLen != udf16Chars) { + throwUtf8DecodedSizeMismatch(udf16Chars, readLen); + } } return newCharsStringZeroCopy(chars); } @@ -724,6 +764,16 @@ private static void checkUtf16Bytes(int numBytes) { } } + // The optimized UTF-8 wire path stores encoded bytes plus the decoded UTF-16 byte count. + // Validate the proportional bound before allocating from the decoded count. + private void checkUtf8DecodedBytes(int numBytes, int utf8Bytes) { + checkStringSize(utf8Bytes); + if ((numBytes & 1) != 0 || numBytes > ((long) utf8Bytes << 1)) { + throw new DeserializationException( + "Invalid UTF-8 decoded byte size " + numBytes + " for encoded byte size " + utf8Bytes); + } + } + private void checkStringSize(int size) { if (size < 0) { throwStringSizeOutOfBounds(size); @@ -734,6 +784,11 @@ private void throwStringSizeOutOfBounds(long size) { throw new DeserializationException("Invalid string byte size " + size); } + private void throwUtf8DecodedSizeMismatch(int expected, int actual) { + throw new DeserializationException( + "UTF-8 decoded size mismatch: expected " + expected + " but got " + actual); + } + public void writeCharsLatin1(MemoryBuffer buffer, char[] chars, int numBytes) { int writerIndex = buffer.writerIndex(); long header = ((long) numBytes << 2) | LATIN1; @@ -969,6 +1024,11 @@ public static String newLatin1StringZeroCopy(byte[] data) { return newBytesStringZeroCopy(LATIN1, data); } + @Internal + public static String newUtf16StringZeroCopy(byte[] data) { + return newBytesStringZeroCopy(UTF16, data); + } + private static String newBytesStringSlow(byte coder, byte[] data) { if (coder == LATIN1) { return new String(data, StandardCharsets.ISO_8859_1); diff --git a/java/fory-core/src/test/java/org/apache/fory/serializer/StringSerializerTest.java b/java/fory-core/src/test/java/org/apache/fory/serializer/StringSerializerTest.java index c910414959..bfa34aa59e 100644 --- a/java/fory-core/src/test/java/org/apache/fory/serializer/StringSerializerTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/serializer/StringSerializerTest.java @@ -33,6 +33,7 @@ import org.apache.fory.Fory; import org.apache.fory.ForyTestBase; import org.apache.fory.collection.Tuple2; +import org.apache.fory.exception.DeserializationException; import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.memory.MemoryUtils; import org.apache.fory.platform.JdkVersion; @@ -456,6 +457,87 @@ public void testReadUtf8String(boolean writeNumUtf16BytesForUtf8Encoding) { } } + @Test + public void testRejectInvalidUtf8DecodedBytes() { + Fory fory = + Fory.builder() + .withXlang(false) + .withStringCompressed(true) + .withWriteNumUtf16BytesForUtf8Encoding(true) + .requireClassRegistration(false) + .withCompatible(false) + .build(); + StringSerializer serializer = new StringSerializer(fory.getConfig()); + byte utf8 = 2; + for (MemoryBuffer buffer : + new MemoryBuffer[] { + MemoryUtils.buffer(32), MemoryUtils.wrap(ByteBuffer.allocateDirect(32)) + }) { + buffer.writerIndex(0); + buffer.readerIndex(0); + buffer.writeVarUInt64(((long) (1 << 20) << 2) | utf8); + buffer.writeInt32(1); + buffer.writeByte((byte) 'a'); + buffer.readerIndex(0); + Assert.assertThrows( + DeserializationException.class, () -> serializer.readCompressedBytesString(buffer)); + } + } + + @Test + public void testRejectMismatchedUtf8DecodedBytes() { + Fory fory = + Fory.builder() + .withXlang(false) + .withStringCompressed(true) + .withWriteNumUtf16BytesForUtf8Encoding(true) + .requireClassRegistration(false) + .withCompatible(false) + .build(); + StringSerializer serializer = new StringSerializer(fory.getConfig()); + byte[] bytes = "éé".getBytes(StandardCharsets.UTF_8); + byte utf8 = 2; + for (MemoryBuffer buffer : + new MemoryBuffer[] { + MemoryUtils.buffer(32), MemoryUtils.wrap(ByteBuffer.allocateDirect(32)) + }) { + buffer.writerIndex(0); + buffer.readerIndex(0); + buffer.writeVarUInt64((6L << 2) | utf8); + buffer.writeInt32(bytes.length); + buffer.writeBytes(bytes); + buffer.readerIndex(0); + Assert.assertThrows( + DeserializationException.class, () -> serializer.readCompressedBytesString(buffer)); + } + } + + @Test + public void testRejectMismatchedUtf8DecodedChars() { + Fory fory = + Fory.builder() + .withXlang(false) + .withStringCompressed(true) + .withWriteNumUtf16BytesForUtf8Encoding(true) + .requireClassRegistration(false) + .withCompatible(false) + .build(); + StringSerializer serializer = new StringSerializer(fory.getConfig()); + byte[] bytes = "éé".getBytes(StandardCharsets.UTF_8); + for (MemoryBuffer buffer : + new MemoryBuffer[] { + MemoryUtils.buffer(32), MemoryUtils.wrap(ByteBuffer.allocateDirect(32)) + }) { + buffer.writerIndex(0); + buffer.readerIndex(0); + buffer.writeInt32(bytes.length); + buffer.writeBytes(bytes); + buffer.readerIndex(0); + Assert.assertThrows( + DeserializationException.class, () -> serializer.readCharsUTF8PerfOptimized(buffer, 6)); + } + } + /** * Comprehensive tests for readBytesUTF8ForXlang method. Tests the optimized single-pass UTF-8 to * Latin1/UTF-16 conversion. diff --git a/java/fory-json/src/main/java/org/apache/fory/json/ForyJson.java b/java/fory-json/src/main/java/org/apache/fory/json/ForyJson.java index 8fc4662da3..608a5da187 100644 --- a/java/fory-json/src/main/java/org/apache/fory/json/ForyJson.java +++ b/java/fory-json/src/main/java/org/apache/fory/json/ForyJson.java @@ -19,14 +19,15 @@ package org.apache.fory.json; +import java.io.OutputStream; import java.lang.reflect.Type; +import java.util.Objects; import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.atomic.AtomicReferenceArray; import org.apache.fory.json.codec.GeneratedObjectCodec; import org.apache.fory.json.reader.Latin1JsonReader; import org.apache.fory.json.reader.Utf16JsonReader; import org.apache.fory.json.reader.Utf8JsonReader; -import org.apache.fory.json.resolver.CodecRegistry; import org.apache.fory.json.resolver.JsonSharedRegistry; import org.apache.fory.json.resolver.JsonTypeInfo; import org.apache.fory.json.resolver.JsonTypeResolver; @@ -39,8 +40,10 @@ public final class ForyJson { private static final int PREFERRED_SLOT_RETRIES = 2; private static final int INITIAL_BUFFER_SIZE = 8192; + private static final int RETAINED_UTF16_BYTES = 64 * 1024; private static final int PRIMARY_SLOT = -1; private static final int TEMPORARY_SLOT = -2; + private static final byte[] EMPTY_BYTES = new byte[0]; private static final int DEFAULT_POOL_SIZE = Math.max(1, Runtime.getRuntime().availableProcessors() * 4); @@ -54,11 +57,10 @@ public final class ForyJson { private final AtomicReference primarySlot; private final AtomicReferenceArray slots; - ForyJson( - boolean writeNullFields, boolean codegenEnabled, int maxDepth, CodecRegistry codecRegistry) { - this.writeNullFields = writeNullFields; - this.maxDepth = maxDepth; - sharedRegistry = new JsonSharedRegistry(codegenEnabled, writeNullFields, codecRegistry); + ForyJson(JsonConfig config) { + this.writeNullFields = config.writeNullFields(); + this.maxDepth = config.maxDepth(); + sharedRegistry = new JsonSharedRegistry(config); poolSize = DEFAULT_POOL_SIZE; primarySlot = new AtomicReference<>( @@ -111,13 +113,34 @@ public byte[] toJsonBytes(Object value) { } } + /** Serializes {@code value} as UTF-8 JSON to {@code output}. */ + public void writeJsonTo(Object value, OutputStream output) { + Objects.requireNonNull(output, "output"); + PooledState entry = acquire(); + JsonState state = entry.state; + Utf8JsonWriter writer = state.utf8Writer; + try { + if (value == null) { + writer.writeNull(); + } else { + JsonTypeResolver resolver = state.typeResolver; + JsonTypeInfo typeInfo = state.rootTypeInfo(value.getClass()); + typeInfo.codec().writeUtf8(writer, value, resolver); + } + writer.writeTo(output); + } finally { + writer.reset(); + release(entry); + } + } + public T fromJson(String json, Class type) { PooledState entry = acquire(); JsonState state = entry.state; try { return castValue(readJavaStringValue(json, type, type, state), type); } finally { - state.clearReaders(); + state.clearStringReaders(); release(entry); } } @@ -130,7 +153,7 @@ public T fromJson(String json, TypeRef typeRef) { Object value = readJavaStringValue(json, typeRef.getType(), typeRef.getRawType(), state); return castValue(value, typeRef); } finally { - state.clearReaders(); + state.clearStringReaders(); release(entry); } } @@ -141,7 +164,7 @@ public T fromJson(byte[] bytes, Class type) { try { return castValue(readUtf8Value(state.utf8Reader(bytes, maxDepth), type, type, state), type); } finally { - state.clearReaders(); + state.clearUtf8Reader(); release(entry); } } @@ -156,7 +179,7 @@ public T fromJson(byte[] bytes, TypeRef typeRef) { state.utf8Reader(bytes, maxDepth), typeRef.getType(), typeRef.getRawType(), state); return castValue(value, typeRef); } finally { - state.clearReaders(); + state.clearUtf8Reader(); release(entry); } } @@ -235,13 +258,15 @@ private Object readJavaStringValue(String json, Type type, Class fallback, Js if (StringSerializer.isBytesBackedString()) { byte coder = StringSerializer.getStringCoder(json); if (StringSerializer.isLatin1Coder(coder)) { + // Keep String input on its reader owner even when ASCII Latin1 bytes match UTF-8; + // custom JsonCodec implementations can observe readLatin1/readUtf8 dispatch. return readLatin1Value(state.latin1Reader(json, maxDepth), type, fallback, state); } if (StringSerializer.isUtf16Coder(coder)) { return readUtf16Value(state.utf16Reader(json, maxDepth), type, fallback, state); } } - return readUtf16Value(state.utf16Reader(json, maxDepth), type, fallback, state); + return readUtf16Value(state.legacyUtf16Reader(json, maxDepth), type, fallback, state); } private Object readLatin1Value( @@ -299,6 +324,7 @@ private static final class JsonState { private final Latin1JsonReader latin1Reader; private final Utf16JsonReader utf16Reader; private final JsonTypeResolver typeResolver; + private byte[] legacyUtf16Bytes; private Type lastRootType; private Class lastRootFallback; private JsonTypeInfo lastRootInfo; @@ -310,6 +336,7 @@ private JsonState(boolean writeNullFields, JsonSharedRegistry sharedRegistry) { latin1Reader = new Latin1JsonReader(); utf16Reader = new Utf16JsonReader(); typeResolver = new JsonTypeResolver(sharedRegistry); + legacyUtf16Bytes = EMPTY_BYTES; } private Latin1JsonReader latin1Reader(String input, int maxDepth) { @@ -324,18 +351,46 @@ private Utf16JsonReader utf16Reader(String input, int maxDepth) { return utf16Reader; } + private Utf16JsonReader legacyUtf16Reader(String input, int maxDepth) { + int length = input.length(); + if (length > (Integer.MAX_VALUE >>> 1)) { + throw new IllegalArgumentException("String is too large"); + } + int numBytes = length << 1; + byte[] bytes; + if (numBytes <= RETAINED_UTF16_BYTES) { + bytes = legacyUtf16Bytes; + if (bytes.length < numBytes) { + bytes = new byte[Math.max(numBytes, INITIAL_BUFFER_SIZE)]; + legacyUtf16Bytes = bytes; + } + } else { + bytes = new byte[numBytes]; + } + // Legacy char[]-backed Strings are converted once so parsing still uses UTF16 byte loads. + StringSerializer.copyStringCharsToBytes(input, bytes); + utf16Reader.resetUtf16Bytes(input, bytes); + utf16Reader.resetDepth(maxDepth); + return utf16Reader; + } + private Utf8JsonReader utf8Reader(byte[] input, int maxDepth) { utf8Reader.reset(input); utf8Reader.resetDepth(maxDepth); return utf8Reader; } - private void clearReaders() { + // Clear only readers reset by the current public parse entry; clearing the unused readers shows + // up on small byte-input parses and does not release additional retained input. + private void clearStringReaders() { latin1Reader.clearDepth(); utf16Reader.clearDepth(); - utf8Reader.clearDepth(); latin1Reader.clear(); utf16Reader.clear(); + } + + private void clearUtf8Reader() { + utf8Reader.clearDepth(); utf8Reader.clear(); } diff --git a/java/fory-json/src/main/java/org/apache/fory/json/ForyJsonBuilder.java b/java/fory-json/src/main/java/org/apache/fory/json/ForyJsonBuilder.java index 17a5cb6c53..148c24edde 100644 --- a/java/fory-json/src/main/java/org/apache/fory/json/ForyJsonBuilder.java +++ b/java/fory-json/src/main/java/org/apache/fory/json/ForyJsonBuilder.java @@ -26,6 +26,7 @@ public final class ForyJsonBuilder { private boolean writeNullFields; private boolean codegenEnabled = true; + private boolean propertyDiscoveryEnabled = true; private int maxDepth = ForyJson.DEFAULT_MAX_DEPTH; private final CodecRegistry codecRegistry = new CodecRegistry(); @@ -43,6 +44,16 @@ public ForyJsonBuilder withCodegen(boolean codegenEnabled) { return this; } + /** + * Enables field mode, where JSON object members are discovered from Java fields only. When + * disabled, Fory JSON uses the default JavaBean property model: public getters, public setters, + * and eligible fields are merged as JSON object members. + */ + public ForyJsonBuilder withFieldMode(boolean fieldMode) { + this.propertyDiscoveryEnabled = !fieldMode; + return this; + } + /** Sets the maximum nested JSON object/array depth allowed while parsing. */ public ForyJsonBuilder maxDepth(int maxDepth) { if (maxDepth < 1) { @@ -59,6 +70,8 @@ public ForyJsonBuilder registerCodec(Class type, JsonCodec codec) { } public ForyJson build() { - return new ForyJson(writeNullFields, codegenEnabled, maxDepth, codecRegistry); + return new ForyJson( + new JsonConfig( + writeNullFields, codegenEnabled, propertyDiscoveryEnabled, maxDepth, codecRegistry)); } } diff --git a/java/fory-json/src/main/java/org/apache/fory/json/JsonConfig.java b/java/fory-json/src/main/java/org/apache/fory/json/JsonConfig.java new file mode 100644 index 0000000000..3cda718a47 --- /dev/null +++ b/java/fory-json/src/main/java/org/apache/fory/json/JsonConfig.java @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.json; + +import java.util.Objects; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicInteger; +import org.apache.fory.json.resolver.CodecRegistry; + +/** Immutable build-time configuration for one {@link ForyJson} instance. */ +public final class JsonConfig { + private final boolean writeNullFields; + private final boolean codegenEnabled; + private final boolean propertyDiscoveryEnabled; + private final int maxDepth; + private final CodecRegistry codecRegistry; + private final String codecRegistryKey; + private transient int configHash; + + JsonConfig( + boolean writeNullFields, + boolean codegenEnabled, + boolean propertyDiscoveryEnabled, + int maxDepth, + CodecRegistry codecRegistry) { + this.writeNullFields = writeNullFields; + this.codegenEnabled = codegenEnabled; + this.propertyDiscoveryEnabled = propertyDiscoveryEnabled; + this.maxDepth = maxDepth; + this.codecRegistry = codecRegistry; + codecRegistryKey = codecRegistry.codegenKey(); + } + + public boolean writeNullFields() { + return writeNullFields; + } + + public boolean codegenEnabled() { + return codegenEnabled; + } + + public boolean propertyDiscoveryEnabled() { + return propertyDiscoveryEnabled; + } + + public int maxDepth() { + return maxDepth; + } + + public CodecRegistry codecRegistry() { + return codecRegistry; + } + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + if (other == null || getClass() != other.getClass()) { + return false; + } + JsonConfig that = (JsonConfig) other; + return writeNullFields == that.writeNullFields + && codegenEnabled == that.codegenEnabled + && propertyDiscoveryEnabled == that.propertyDiscoveryEnabled + && maxDepth == that.maxDepth + && Objects.equals(codecRegistryKey, that.codecRegistryKey); + } + + @Override + public int hashCode() { + return Objects.hash( + writeNullFields, codegenEnabled, propertyDiscoveryEnabled, maxDepth, codecRegistryKey); + } + + private static final AtomicInteger COUNTER = new AtomicInteger(0); + + // Equal configs share one map entry, following core Config's generated-code naming model. + private static final ConcurrentMap CONFIG_ID_MAP = new ConcurrentHashMap<>(); + + public int getConfigHash() { + if (configHash == 0) { + configHash = CONFIG_ID_MAP.computeIfAbsent(this, key -> COUNTER.incrementAndGet()); + } + return configHash; + } +} diff --git a/java/fory-json/src/main/java/org/apache/fory/json/codec/ArrayCodec.java b/java/fory-json/src/main/java/org/apache/fory/json/codec/ArrayCodec.java index 9724abbd92..1f8fb59ceb 100644 --- a/java/fory-json/src/main/java/org/apache/fory/json/codec/ArrayCodec.java +++ b/java/fory-json/src/main/java/org/apache/fory/json/codec/ArrayCodec.java @@ -20,9 +20,7 @@ package org.apache.fory.json.codec; import java.lang.reflect.Array; -import java.util.ArrayList; import java.util.Arrays; -import java.util.List; import org.apache.fory.json.ForyJsonException; import org.apache.fory.json.reader.JsonReader; import org.apache.fory.json.reader.Latin1JsonReader; @@ -58,6 +56,24 @@ public static ArrayCodec create(Class componentType, JsonTypeResolver resolve return FloatArrayCodec.INSTANCE; } else if (componentType == double.class) { return DoubleArrayCodec.INSTANCE; + } else if (componentType == Integer.class) { + return BoxedIntArrayCodec.INSTANCE; + } else if (componentType == Long.class) { + return BoxedLongArrayCodec.INSTANCE; + } else if (componentType == Boolean.class) { + return BoxedBooleanArrayCodec.INSTANCE; + } else if (componentType == Short.class) { + return BoxedShortArrayCodec.INSTANCE; + } else if (componentType == Byte.class) { + return BoxedByteArrayCodec.INSTANCE; + } else if (componentType == Character.class) { + return BoxedCharArrayCodec.INSTANCE; + } else if (componentType == Float.class) { + return BoxedFloatArrayCodec.INSTANCE; + } else if (componentType == Double.class) { + return BoxedDoubleArrayCodec.INSTANCE; + } else if (componentType == String.class) { + return StringArrayCodec.INSTANCE; } return new ObjectArrayCodec(componentType, resolver.getTypeInfo(componentType, componentType)); } @@ -243,16 +259,88 @@ public Object readLatin1( reader.exitDepth(); return new long[0]; } - long[] values = new long[8]; - int size = 0; + rejectNull(reader); + long v0 = reader.readLongValue(); + if (!reader.consumeNextCommaOrEndArray()) { + reader.exitDepth(); + return new long[] {v0}; + } + rejectNull(reader); + long v1 = reader.readLongValue(); + if (!reader.consumeNextCommaOrEndArray()) { + reader.exitDepth(); + return new long[] {v0, v1}; + } + rejectNull(reader); + long v2 = reader.readLongValue(); + if (!reader.consumeNextCommaOrEndArray()) { + reader.exitDepth(); + return new long[] {v0, v1, v2}; + } + rejectNull(reader); + long v3 = reader.readLongValue(); + if (!reader.consumeNextCommaOrEndArray()) { + reader.exitDepth(); + return new long[] {v0, v1, v2, v3}; + } + return readLatin1Tail(reader, v0, v1, v2, v3); + } + + private long[] readLatin1Tail(Latin1JsonReader reader, long v0, long v1, long v2, long v3) { + rejectNull(reader); + long v4 = reader.readLongValue(); + if (!reader.consumeNextCommaOrEndArray()) { + reader.exitDepth(); + return new long[] {v0, v1, v2, v3, v4}; + } + rejectNull(reader); + long v5 = reader.readLongValue(); + if (!reader.consumeNextCommaOrEndArray()) { + reader.exitDepth(); + return new long[] {v0, v1, v2, v3, v4, v5}; + } + rejectNull(reader); + long v6 = reader.readLongValue(); + if (!reader.consumeNextCommaOrEndArray()) { + reader.exitDepth(); + return new long[] {v0, v1, v2, v3, v4, v5, v6}; + } + rejectNull(reader); + long v7 = reader.readLongValue(); + if (!reader.consumeNextCommaOrEndArray()) { + reader.exitDepth(); + return new long[] {v0, v1, v2, v3, v4, v5, v6, v7}; + } + return readLatin1LongTail(reader, v0, v1, v2, v3, v4, v5, v6, v7); + } + + private long[] readLatin1LongTail( + Latin1JsonReader reader, + long v0, + long v1, + long v2, + long v3, + long v4, + long v5, + long v6, + long v7) { + long[] values = new long[16]; + values[0] = v0; + values[1] = v1; + values[2] = v2; + values[3] = v3; + values[4] = v4; + values[5] = v5; + values[6] = v6; + values[7] = v7; + int size = 8; do { rejectNull(reader); if (size == values.length) { values = Arrays.copyOf(values, values.length << 1); } values[size++] = reader.readLongValue(); - } while (reader.consumeNextToken(',')); - reader.expectNextToken(']'); + } while (reader.consumeNextCommaOrEndArray()); reader.exitDepth(); return Arrays.copyOf(values, size); } @@ -269,16 +357,88 @@ public Object readUtf16( reader.exitDepth(); return new long[0]; } - long[] values = new long[8]; - int size = 0; + rejectNull(reader); + long v0 = reader.readLongValue(); + if (!reader.consumeNextCommaOrEndArray()) { + reader.exitDepth(); + return new long[] {v0}; + } + rejectNull(reader); + long v1 = reader.readLongValue(); + if (!reader.consumeNextCommaOrEndArray()) { + reader.exitDepth(); + return new long[] {v0, v1}; + } + rejectNull(reader); + long v2 = reader.readLongValue(); + if (!reader.consumeNextCommaOrEndArray()) { + reader.exitDepth(); + return new long[] {v0, v1, v2}; + } + rejectNull(reader); + long v3 = reader.readLongValue(); + if (!reader.consumeNextCommaOrEndArray()) { + reader.exitDepth(); + return new long[] {v0, v1, v2, v3}; + } + return readUtf16Tail(reader, v0, v1, v2, v3); + } + + private long[] readUtf16Tail(Utf16JsonReader reader, long v0, long v1, long v2, long v3) { + rejectNull(reader); + long v4 = reader.readLongValue(); + if (!reader.consumeNextCommaOrEndArray()) { + reader.exitDepth(); + return new long[] {v0, v1, v2, v3, v4}; + } + rejectNull(reader); + long v5 = reader.readLongValue(); + if (!reader.consumeNextCommaOrEndArray()) { + reader.exitDepth(); + return new long[] {v0, v1, v2, v3, v4, v5}; + } + rejectNull(reader); + long v6 = reader.readLongValue(); + if (!reader.consumeNextCommaOrEndArray()) { + reader.exitDepth(); + return new long[] {v0, v1, v2, v3, v4, v5, v6}; + } + rejectNull(reader); + long v7 = reader.readLongValue(); + if (!reader.consumeNextCommaOrEndArray()) { + reader.exitDepth(); + return new long[] {v0, v1, v2, v3, v4, v5, v6, v7}; + } + return readUtf16LongTail(reader, v0, v1, v2, v3, v4, v5, v6, v7); + } + + private long[] readUtf16LongTail( + Utf16JsonReader reader, + long v0, + long v1, + long v2, + long v3, + long v4, + long v5, + long v6, + long v7) { + long[] values = new long[16]; + values[0] = v0; + values[1] = v1; + values[2] = v2; + values[3] = v3; + values[4] = v4; + values[5] = v5; + values[6] = v6; + values[7] = v7; + int size = 8; do { rejectNull(reader); if (size == values.length) { values = Arrays.copyOf(values, values.length << 1); } values[size++] = reader.readLongValue(); - } while (reader.consumeNextToken(',')); - reader.expectNextToken(']'); + } while (reader.consumeNextCommaOrEndArray()); reader.exitDepth(); return Arrays.copyOf(values, size); } @@ -295,16 +455,90 @@ public Object readUtf8( reader.exitDepth(); return new long[0]; } - long[] values = new long[8]; - int size = 0; + rejectNull(reader); + long v0 = reader.readLongValue(); + if (!reader.consumeNextCommaOrEndArray()) { + reader.exitDepth(); + return new long[] {v0}; + } + rejectNull(reader); + long v1 = reader.readLongValue(); + if (!reader.consumeNextCommaOrEndArray()) { + reader.exitDepth(); + return new long[] {v0, v1}; + } + rejectNull(reader); + long v2 = reader.readLongValue(); + if (!reader.consumeNextCommaOrEndArray()) { + reader.exitDepth(); + return new long[] {v0, v1, v2}; + } + rejectNull(reader); + long v3 = reader.readLongValue(); + if (!reader.consumeNextCommaOrEndArray()) { + reader.exitDepth(); + return new long[] {v0, v1, v2, v3}; + } + return readUtf8Tail(reader, v0, v1, v2, v3); + } + + private long[] readUtf8Tail(Utf8JsonReader reader, long v0, long v1, long v2, long v3) { + rejectNull(reader); + long v4 = reader.readLongValue(); + if (!reader.consumeNextCommaOrEndArray()) { + reader.exitDepth(); + return new long[] {v0, v1, v2, v3, v4}; + } + rejectNull(reader); + long v5 = reader.readLongValue(); + if (!reader.consumeNextCommaOrEndArray()) { + reader.exitDepth(); + return new long[] {v0, v1, v2, v3, v4, v5}; + } + rejectNull(reader); + long v6 = reader.readLongValue(); + if (!reader.consumeNextCommaOrEndArray()) { + reader.exitDepth(); + return new long[] {v0, v1, v2, v3, v4, v5, v6}; + } + rejectNull(reader); + long v7 = reader.readLongValue(); + if (!reader.consumeNextCommaOrEndArray()) { + reader.exitDepth(); + return new long[] {v0, v1, v2, v3, v4, v5, v6, v7}; + } + return readUtf8LongTail(reader, v0, v1, v2, v3, v4, v5, v6, v7); + } + + // Keep dynamic growth out of the exact small-array path so C2 can inline the common cases + // without pulling the uncommon grow/copy loop into the caller's inline budget. + private long[] readUtf8LongTail( + Utf8JsonReader reader, + long v0, + long v1, + long v2, + long v3, + long v4, + long v5, + long v6, + long v7) { + long[] values = new long[16]; + values[0] = v0; + values[1] = v1; + values[2] = v2; + values[3] = v3; + values[4] = v4; + values[5] = v5; + values[6] = v6; + values[7] = v7; + int size = 8; do { rejectNull(reader); if (size == values.length) { values = Arrays.copyOf(values, values.length << 1); } values[size++] = reader.readLongValue(); - } while (reader.consumeNextToken(',')); - reader.expectNextToken(']'); + } while (reader.consumeNextCommaOrEndArray()); reader.exitDepth(); return Arrays.copyOf(values, size); } @@ -634,9 +868,373 @@ Object readNonNull(JsonReader reader, JsonTypeInfo typeInfo, JsonTypeResolver re } } + public static final class StringArrayCodec extends ArrayCodec { + private static final StringArrayCodec INSTANCE = new StringArrayCodec(); + + private StringArrayCodec() { + super(String.class); + } + + @Override + void writeNonNull(JsonWriter writer, Object value, JsonTypeResolver resolver) { + String[] array = (String[]) value; + writer.writeArrayStart(); + for (int i = 0; i < array.length; i++) { + writer.writeComma(i); + if (array[i] == null) { + writer.writeNull(); + } else { + writer.writeString(array[i]); + } + } + writer.writeArrayEnd(); + } + + @Override + void writeStringNonNull(StringJsonWriter writer, Object value, JsonTypeResolver resolver) { + String[] array = (String[]) value; + writer.writeArrayStart(); + for (int i = 0; i < array.length; i++) { + writer.writeStringElement(i, array[i]); + } + writer.writeArrayEnd(); + } + + @Override + void writeUtf8NonNull(Utf8JsonWriter writer, Object value, JsonTypeResolver resolver) { + String[] array = (String[]) value; + writer.writeArrayStart(); + for (int i = 0; i < array.length; i++) { + writer.writeStringElement(i, array[i]); + } + writer.writeArrayEnd(); + } + + @Override + Object readNonNull(JsonReader reader, JsonTypeInfo typeInfo, JsonTypeResolver resolver) { + reader.enterDepth(); + reader.expect('['); + if (reader.consume(']')) { + reader.exitDepth(); + return new String[0]; + } + String[] values = new String[8]; + int size = 0; + do { + if (size == values.length) { + values = Arrays.copyOf(values, values.length << 1); + } + values[size++] = reader.readNullableString(); + } while (reader.consume(',')); + reader.expect(']'); + reader.exitDepth(); + return Arrays.copyOf(values, size); + } + + @Override + public Object readLatin1( + Latin1JsonReader reader, JsonTypeInfo typeInfo, JsonTypeResolver resolver) { + if (reader.tryReadNullToken()) { + return null; + } + reader.enterDepth(); + reader.expectNextToken('['); + if (reader.consumeNextToken(']')) { + reader.exitDepth(); + return new String[0]; + } + String v0 = reader.readNextNullableString(); + if (!reader.consumeNextCommaOrEndArray()) { + reader.exitDepth(); + return new String[] {v0}; + } + String v1 = reader.readNextNullableString(); + if (!reader.consumeNextCommaOrEndArray()) { + reader.exitDepth(); + return new String[] {v0, v1}; + } + String v2 = reader.readNextNullableString(); + if (!reader.consumeNextCommaOrEndArray()) { + reader.exitDepth(); + return new String[] {v0, v1, v2}; + } + String v3 = reader.readNextNullableString(); + if (!reader.consumeNextCommaOrEndArray()) { + reader.exitDepth(); + return new String[] {v0, v1, v2, v3}; + } + return readLatin1Tail(reader, v0, v1, v2, v3); + } + + private String[] readLatin1Tail( + Latin1JsonReader reader, String v0, String v1, String v2, String v3) { + String v4 = reader.readNextNullableString(); + if (!reader.consumeNextCommaOrEndArray()) { + reader.exitDepth(); + return new String[] {v0, v1, v2, v3, v4}; + } + String v5 = reader.readNextNullableString(); + if (!reader.consumeNextCommaOrEndArray()) { + reader.exitDepth(); + return new String[] {v0, v1, v2, v3, v4, v5}; + } + String v6 = reader.readNextNullableString(); + if (!reader.consumeNextCommaOrEndArray()) { + reader.exitDepth(); + return new String[] {v0, v1, v2, v3, v4, v5, v6}; + } + String v7 = reader.readNextNullableString(); + if (!reader.consumeNextCommaOrEndArray()) { + reader.exitDepth(); + return new String[] {v0, v1, v2, v3, v4, v5, v6, v7}; + } + String v8 = reader.readNextNullableString(); + if (!reader.consumeNextCommaOrEndArray()) { + reader.exitDepth(); + return new String[] {v0, v1, v2, v3, v4, v5, v6, v7, v8}; + } + return readLatin1LongTail(reader, v0, v1, v2, v3, v4, v5, v6, v7, v8); + } + + private String[] readLatin1LongTail( + Latin1JsonReader reader, + String v0, + String v1, + String v2, + String v3, + String v4, + String v5, + String v6, + String v7, + String v8) { + String[] values = new String[16]; + values[0] = v0; + values[1] = v1; + values[2] = v2; + values[3] = v3; + values[4] = v4; + values[5] = v5; + values[6] = v6; + values[7] = v7; + values[8] = v8; + int size = 9; + do { + if (size == values.length) { + values = Arrays.copyOf(values, values.length << 1); + } + values[size++] = reader.readNextNullableString(); + } while (reader.consumeNextCommaOrEndArray()); + reader.exitDepth(); + return Arrays.copyOf(values, size); + } + + @Override + public Object readUtf16( + Utf16JsonReader reader, JsonTypeInfo typeInfo, JsonTypeResolver resolver) { + if (reader.tryReadNullToken()) { + return null; + } + reader.enterDepth(); + reader.expectNextToken('['); + if (reader.consumeNextToken(']')) { + reader.exitDepth(); + return new String[0]; + } + String v0 = reader.readNextNullableString(); + if (!reader.consumeNextCommaOrEndArray()) { + reader.exitDepth(); + return new String[] {v0}; + } + String v1 = reader.readNextNullableString(); + if (!reader.consumeNextCommaOrEndArray()) { + reader.exitDepth(); + return new String[] {v0, v1}; + } + String v2 = reader.readNextNullableString(); + if (!reader.consumeNextCommaOrEndArray()) { + reader.exitDepth(); + return new String[] {v0, v1, v2}; + } + String v3 = reader.readNextNullableString(); + if (!reader.consumeNextCommaOrEndArray()) { + reader.exitDepth(); + return new String[] {v0, v1, v2, v3}; + } + return readUtf16Tail(reader, v0, v1, v2, v3); + } + + private String[] readUtf16Tail( + Utf16JsonReader reader, String v0, String v1, String v2, String v3) { + String v4 = reader.readNextNullableString(); + if (!reader.consumeNextCommaOrEndArray()) { + reader.exitDepth(); + return new String[] {v0, v1, v2, v3, v4}; + } + String v5 = reader.readNextNullableString(); + if (!reader.consumeNextCommaOrEndArray()) { + reader.exitDepth(); + return new String[] {v0, v1, v2, v3, v4, v5}; + } + String v6 = reader.readNextNullableString(); + if (!reader.consumeNextCommaOrEndArray()) { + reader.exitDepth(); + return new String[] {v0, v1, v2, v3, v4, v5, v6}; + } + String v7 = reader.readNextNullableString(); + if (!reader.consumeNextCommaOrEndArray()) { + reader.exitDepth(); + return new String[] {v0, v1, v2, v3, v4, v5, v6, v7}; + } + String v8 = reader.readNextNullableString(); + if (!reader.consumeNextCommaOrEndArray()) { + reader.exitDepth(); + return new String[] {v0, v1, v2, v3, v4, v5, v6, v7, v8}; + } + return readUtf16LongTail(reader, v0, v1, v2, v3, v4, v5, v6, v7, v8); + } + + private String[] readUtf16LongTail( + Utf16JsonReader reader, + String v0, + String v1, + String v2, + String v3, + String v4, + String v5, + String v6, + String v7, + String v8) { + String[] values = new String[16]; + values[0] = v0; + values[1] = v1; + values[2] = v2; + values[3] = v3; + values[4] = v4; + values[5] = v5; + values[6] = v6; + values[7] = v7; + values[8] = v8; + int size = 9; + do { + if (size == values.length) { + values = Arrays.copyOf(values, values.length << 1); + } + values[size++] = reader.readNextNullableString(); + } while (reader.consumeNextCommaOrEndArray()); + reader.exitDepth(); + return Arrays.copyOf(values, size); + } + + @Override + public Object readUtf8( + Utf8JsonReader reader, JsonTypeInfo typeInfo, JsonTypeResolver resolver) { + if (reader.tryReadNullToken()) { + return null; + } + reader.enterDepth(); + reader.expectNextToken('['); + if (reader.consumeNextToken(']')) { + reader.exitDepth(); + return new String[0]; + } + String v0 = reader.readNextNullableString(); + if (!reader.consumeNextCommaOrEndArray()) { + reader.exitDepth(); + return new String[] {v0}; + } + String v1 = reader.readNextNullableString(); + if (!reader.consumeNextCommaOrEndArray()) { + reader.exitDepth(); + return new String[] {v0, v1}; + } + String v2 = reader.readNextNullableString(); + if (!reader.consumeNextCommaOrEndArray()) { + reader.exitDepth(); + return new String[] {v0, v1, v2}; + } + String v3 = reader.readNextNullableString(); + if (!reader.consumeNextCommaOrEndArray()) { + reader.exitDepth(); + return new String[] {v0, v1, v2, v3}; + } + return readUtf8Tail(reader, v0, v1, v2, v3); + } + + private String[] readUtf8Tail( + Utf8JsonReader reader, String v0, String v1, String v2, String v3) { + String v4 = reader.readNextNullableString(); + if (!reader.consumeNextCommaOrEndArray()) { + reader.exitDepth(); + return new String[] {v0, v1, v2, v3, v4}; + } + String v5 = reader.readNextNullableString(); + if (!reader.consumeNextCommaOrEndArray()) { + reader.exitDepth(); + return new String[] {v0, v1, v2, v3, v4, v5}; + } + String v6 = reader.readNextNullableString(); + if (!reader.consumeNextCommaOrEndArray()) { + reader.exitDepth(); + return new String[] {v0, v1, v2, v3, v4, v5, v6}; + } + String v7 = reader.readNextNullableString(); + if (!reader.consumeNextCommaOrEndArray()) { + reader.exitDepth(); + return new String[] {v0, v1, v2, v3, v4, v5, v6, v7}; + } + String v8 = reader.readNextNullableString(); + if (!reader.consumeNextCommaOrEndArray()) { + reader.exitDepth(); + return new String[] {v0, v1, v2, v3, v4, v5, v6, v7, v8}; + } + return readUtf8LongTail(reader, v0, v1, v2, v3, v4, v5, v6, v7, v8); + } + + // Keep dynamic growth out of the exact small-array path so C2 can inline the common cases + // without pulling the uncommon grow/copy loop into the caller's inline budget. + private String[] readUtf8LongTail( + Utf8JsonReader reader, + String v0, + String v1, + String v2, + String v3, + String v4, + String v5, + String v6, + String v7, + String v8) { + String[] values = new String[16]; + values[0] = v0; + values[1] = v1; + values[2] = v2; + values[3] = v3; + values[4] = v4; + values[5] = v5; + values[6] = v6; + values[7] = v7; + values[8] = v8; + int size = 9; + do { + if (size == values.length) { + values = Arrays.copyOf(values, values.length << 1); + } + values[size++] = reader.readNextNullableString(); + } while (reader.consumeNextCommaOrEndArray()); + reader.exitDepth(); + return Arrays.copyOf(values, size); + } + } + public static final class ObjectArrayCodec extends ArrayCodec { + private static final int VALUES_CACHE_DEPTH = 8; + private static final int INITIAL_VALUES_SIZE = 8; + private static final int MAX_CACHED_VALUES_SIZE = 1024; + private final JsonTypeInfo elementTypeInfo; private final JsonCodec elementCodec; + // Recursive object-array reads borrow one scratch slot per active depth. + private final Object[][] valuesCache = new Object[VALUES_CACHE_DEPTH][]; + private int valuesDepth; private ObjectArrayCodec(Class componentType, JsonTypeInfo elementTypeInfo) { super(componentType); @@ -680,24 +1278,410 @@ void writeUtf8NonNull(Utf8JsonWriter writer, Object value, JsonTypeResolver reso @Override Object readNonNull(JsonReader reader, JsonTypeInfo typeInfo, JsonTypeResolver resolver) { reader.enterDepth(); - List values = new ArrayList<>(0); + int depth = valuesDepth; + boolean useCache = depth < VALUES_CACHE_DEPTH; + Object[] values = null; + if (useCache) { + values = valuesCache[depth]; + valuesCache[depth] = null; + } + if (values == null) { + values = new Object[INITIAL_VALUES_SIZE]; + } + int size = 0; + boolean success = false; + valuesDepth = depth + 1; + try { + reader.expect('['); + if (!reader.consume(']')) { + do { + if (size == values.length) { + values = Arrays.copyOf(values, values.length << 1); + } + values[size++] = elementCodec.read(reader, elementTypeInfo, resolver); + } while (reader.consume(',')); + reader.expect(']'); + } + Object[] array = (Object[]) Array.newInstance(componentType, size); + System.arraycopy(values, 0, array, 0, size); + success = true; + return array; + } finally { + reader.exitDepth(); + // Failed reads drop the scratch array, because it may contain partially parsed user values. + if (success && useCache) { + if (values.length <= MAX_CACHED_VALUES_SIZE) { + Arrays.fill(values, 0, size, null); + valuesCache[depth] = values; + } else { + // Keep the depth slot usable without retaining a grown array from one large value. + valuesCache[depth] = new Object[INITIAL_VALUES_SIZE]; + } + } + // Restore the codec recursion depth after the matching cache slot has been handled. + valuesDepth = depth; + } + } + } + + public static final class BoxedIntArrayCodec extends ArrayCodec { + private static final BoxedIntArrayCodec INSTANCE = new BoxedIntArrayCodec(); + + private BoxedIntArrayCodec() { + super(Integer.class); + } + + @Override + void writeNonNull(JsonWriter writer, Object value, JsonTypeResolver resolver) { + Integer[] array = (Integer[]) value; + writer.writeArrayStart(); + for (int i = 0; i < array.length; i++) { + writer.writeComma(i); + Integer element = array[i]; + if (element == null) { + writer.writeNull(); + } else { + writer.writeInt(element); + } + } + writer.writeArrayEnd(); + } + + @Override + Object readNonNull(JsonReader reader, JsonTypeInfo typeInfo, JsonTypeResolver resolver) { + reader.enterDepth(); + reader.expect('['); + if (reader.consume(']')) { + reader.exitDepth(); + return new Integer[0]; + } + Integer[] values = new Integer[8]; + int size = 0; + do { + if (size == values.length) { + values = Arrays.copyOf(values, values.length << 1); + } + values[size++] = reader.tryReadNull() ? null : reader.readInt(); + } while (reader.consume(',')); + reader.expect(']'); + reader.exitDepth(); + return Arrays.copyOf(values, size); + } + } + + public static final class BoxedLongArrayCodec extends ArrayCodec { + private static final BoxedLongArrayCodec INSTANCE = new BoxedLongArrayCodec(); + + private BoxedLongArrayCodec() { + super(Long.class); + } + + @Override + void writeNonNull(JsonWriter writer, Object value, JsonTypeResolver resolver) { + Long[] array = (Long[]) value; + writer.writeArrayStart(); + for (int i = 0; i < array.length; i++) { + writer.writeComma(i); + Long element = array[i]; + if (element == null) { + writer.writeNull(); + } else { + writer.writeLong(element); + } + } + writer.writeArrayEnd(); + } + + @Override + Object readNonNull(JsonReader reader, JsonTypeInfo typeInfo, JsonTypeResolver resolver) { + reader.enterDepth(); + reader.expect('['); + if (reader.consume(']')) { + reader.exitDepth(); + return new Long[0]; + } + Long[] values = new Long[8]; + int size = 0; + do { + if (size == values.length) { + values = Arrays.copyOf(values, values.length << 1); + } + values[size++] = reader.tryReadNull() ? null : reader.readLong(); + } while (reader.consume(',')); + reader.expect(']'); + reader.exitDepth(); + return Arrays.copyOf(values, size); + } + } + + public static final class BoxedBooleanArrayCodec extends ArrayCodec { + private static final BoxedBooleanArrayCodec INSTANCE = new BoxedBooleanArrayCodec(); + + private BoxedBooleanArrayCodec() { + super(Boolean.class); + } + + @Override + void writeNonNull(JsonWriter writer, Object value, JsonTypeResolver resolver) { + Boolean[] array = (Boolean[]) value; + writer.writeArrayStart(); + for (int i = 0; i < array.length; i++) { + writer.writeComma(i); + Boolean element = array[i]; + if (element == null) { + writer.writeNull(); + } else { + writer.writeBoolean(element); + } + } + writer.writeArrayEnd(); + } + + @Override + Object readNonNull(JsonReader reader, JsonTypeInfo typeInfo, JsonTypeResolver resolver) { + reader.enterDepth(); reader.expect('['); - if (!reader.consume(']')) { - do { - values.add(elementCodec.read(reader, elementTypeInfo, resolver)); - } while (reader.consume(',')); - reader.expect(']'); + if (reader.consume(']')) { + reader.exitDepth(); + return new Boolean[0]; } + Boolean[] values = new Boolean[8]; + int size = 0; + do { + if (size == values.length) { + values = Arrays.copyOf(values, values.length << 1); + } + values[size++] = reader.tryReadNull() ? null : reader.readBoolean(); + } while (reader.consume(',')); + reader.expect(']'); reader.exitDepth(); - return toArray(values); + return Arrays.copyOf(values, size); } + } - private Object toArray(List values) { - Object array = Array.newInstance(componentType, values.size()); - for (int i = 0; i < values.size(); i++) { - Array.set(array, i, values.get(i)); + public static final class BoxedShortArrayCodec extends ArrayCodec { + private static final BoxedShortArrayCodec INSTANCE = new BoxedShortArrayCodec(); + + private BoxedShortArrayCodec() { + super(Short.class); + } + + @Override + void writeNonNull(JsonWriter writer, Object value, JsonTypeResolver resolver) { + Short[] array = (Short[]) value; + writer.writeArrayStart(); + for (int i = 0; i < array.length; i++) { + writer.writeComma(i); + Short element = array[i]; + if (element == null) { + writer.writeNull(); + } else { + writer.writeInt(element); + } } - return array; + writer.writeArrayEnd(); + } + + @Override + Object readNonNull(JsonReader reader, JsonTypeInfo typeInfo, JsonTypeResolver resolver) { + reader.enterDepth(); + reader.expect('['); + if (reader.consume(']')) { + reader.exitDepth(); + return new Short[0]; + } + Short[] values = new Short[8]; + int size = 0; + do { + if (size == values.length) { + values = Arrays.copyOf(values, values.length << 1); + } + values[size++] = reader.tryReadNull() ? null : readShort(reader.readInt()); + } while (reader.consume(',')); + reader.expect(']'); + reader.exitDepth(); + return Arrays.copyOf(values, size); + } + } + + public static final class BoxedByteArrayCodec extends ArrayCodec { + private static final BoxedByteArrayCodec INSTANCE = new BoxedByteArrayCodec(); + + private BoxedByteArrayCodec() { + super(Byte.class); + } + + @Override + void writeNonNull(JsonWriter writer, Object value, JsonTypeResolver resolver) { + Byte[] array = (Byte[]) value; + writer.writeArrayStart(); + for (int i = 0; i < array.length; i++) { + writer.writeComma(i); + Byte element = array[i]; + if (element == null) { + writer.writeNull(); + } else { + writer.writeInt(element); + } + } + writer.writeArrayEnd(); + } + + @Override + Object readNonNull(JsonReader reader, JsonTypeInfo typeInfo, JsonTypeResolver resolver) { + reader.enterDepth(); + reader.expect('['); + if (reader.consume(']')) { + reader.exitDepth(); + return new Byte[0]; + } + Byte[] values = new Byte[8]; + int size = 0; + do { + if (size == values.length) { + values = Arrays.copyOf(values, values.length << 1); + } + values[size++] = reader.tryReadNull() ? null : readByte(reader.readInt()); + } while (reader.consume(',')); + reader.expect(']'); + reader.exitDepth(); + return Arrays.copyOf(values, size); + } + } + + public static final class BoxedCharArrayCodec extends ArrayCodec { + private static final BoxedCharArrayCodec INSTANCE = new BoxedCharArrayCodec(); + + private BoxedCharArrayCodec() { + super(Character.class); + } + + @Override + void writeNonNull(JsonWriter writer, Object value, JsonTypeResolver resolver) { + Character[] array = (Character[]) value; + writer.writeArrayStart(); + for (int i = 0; i < array.length; i++) { + writer.writeComma(i); + Character element = array[i]; + if (element == null) { + writer.writeNull(); + } else { + writer.writeChar(element); + } + } + writer.writeArrayEnd(); + } + + @Override + Object readNonNull(JsonReader reader, JsonTypeInfo typeInfo, JsonTypeResolver resolver) { + reader.enterDepth(); + reader.expect('['); + if (reader.consume(']')) { + reader.exitDepth(); + return new Character[0]; + } + Character[] values = new Character[8]; + int size = 0; + do { + if (size == values.length) { + values = Arrays.copyOf(values, values.length << 1); + } + String value = reader.readNullableString(); + values[size++] = value == null ? null : readChar(value); + } while (reader.consume(',')); + reader.expect(']'); + reader.exitDepth(); + return Arrays.copyOf(values, size); + } + } + + public static final class BoxedFloatArrayCodec extends ArrayCodec { + private static final BoxedFloatArrayCodec INSTANCE = new BoxedFloatArrayCodec(); + + private BoxedFloatArrayCodec() { + super(Float.class); + } + + @Override + void writeNonNull(JsonWriter writer, Object value, JsonTypeResolver resolver) { + Float[] array = (Float[]) value; + writer.writeArrayStart(); + for (int i = 0; i < array.length; i++) { + writer.writeComma(i); + Float element = array[i]; + if (element == null) { + writer.writeNull(); + } else { + writer.writeFloat(element); + } + } + writer.writeArrayEnd(); + } + + @Override + Object readNonNull(JsonReader reader, JsonTypeInfo typeInfo, JsonTypeResolver resolver) { + reader.enterDepth(); + reader.expect('['); + if (reader.consume(']')) { + reader.exitDepth(); + return new Float[0]; + } + Float[] values = new Float[8]; + int size = 0; + do { + if (size == values.length) { + values = Arrays.copyOf(values, values.length << 1); + } + values[size++] = reader.tryReadNull() ? null : Float.parseFloat(reader.readNumber()); + } while (reader.consume(',')); + reader.expect(']'); + reader.exitDepth(); + return Arrays.copyOf(values, size); + } + } + + public static final class BoxedDoubleArrayCodec extends ArrayCodec { + private static final BoxedDoubleArrayCodec INSTANCE = new BoxedDoubleArrayCodec(); + + private BoxedDoubleArrayCodec() { + super(Double.class); + } + + @Override + void writeNonNull(JsonWriter writer, Object value, JsonTypeResolver resolver) { + Double[] array = (Double[]) value; + writer.writeArrayStart(); + for (int i = 0; i < array.length; i++) { + writer.writeComma(i); + Double element = array[i]; + if (element == null) { + writer.writeNull(); + } else { + writer.writeDouble(element); + } + } + writer.writeArrayEnd(); + } + + @Override + Object readNonNull(JsonReader reader, JsonTypeInfo typeInfo, JsonTypeResolver resolver) { + reader.enterDepth(); + reader.expect('['); + if (reader.consume(']')) { + reader.exitDepth(); + return new Double[0]; + } + Double[] values = new Double[8]; + int size = 0; + do { + if (size == values.length) { + values = Arrays.copyOf(values, values.length << 1); + } + values[size++] = reader.tryReadNull() ? null : Double.parseDouble(reader.readNumber()); + } while (reader.consume(',')); + reader.expect(']'); + reader.exitDepth(); + return Arrays.copyOf(values, size); } } @@ -740,7 +1724,10 @@ private static byte readByte(int value) { } private static char readChar(JsonReader reader) { - String value = reader.readString(); + return readChar(reader.readString()); + } + + private static char readChar(String value) { if (value.length() != 1) { throw new ForyJsonException("Expected one-character JSON string for char"); } diff --git a/java/fory-json/src/main/java/org/apache/fory/json/codec/BaseObjectCodec.java b/java/fory-json/src/main/java/org/apache/fory/json/codec/BaseObjectCodec.java index 3566caa8bc..631141b2d1 100644 --- a/java/fory-json/src/main/java/org/apache/fory/json/codec/BaseObjectCodec.java +++ b/java/fory-json/src/main/java/org/apache/fory/json/codec/BaseObjectCodec.java @@ -20,12 +20,14 @@ package org.apache.fory.json.codec; import java.lang.reflect.Field; +import java.lang.reflect.Method; import java.lang.reflect.Modifier; +import java.lang.reflect.Type; import java.util.ArrayList; import java.util.Arrays; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import java.util.TreeMap; import org.apache.fory.annotation.Expose; import org.apache.fory.annotation.Internal; import org.apache.fory.json.ForyJsonException; @@ -81,7 +83,7 @@ record = RecordUtils.isRecord(type); } } - public static ObjectCodec build(Class type) { + public static ObjectCodec build(Class type, boolean propertyDiscoveryEnabled) { if (type.isInterface() || Modifier.isAbstract(type.getModifiers()) || type.isPrimitive() @@ -92,40 +94,22 @@ public static ObjectCodec build(Class type) { boolean record = RecordUtils.isRecord(type); boolean writeExpose = hasWriteExpose(type); boolean readExpose = hasReadExpose(type, record); - TreeMap builders = new TreeMap<>(); - for (Class current = type; - current != null && current != Object.class; - current = current.getSuperclass()) { - for (Field field : current.getDeclaredFields()) { - int modifiers = field.getModifiers(); - if (!isEligibleField(field)) { - continue; - } - boolean write = includeWrite(field, writeExpose); - boolean read = (record || !Modifier.isFinal(modifiers)) && includeRead(field, readExpose); - if (!write && !read) { - continue; - } - FieldBuilder builder = new FieldBuilder(field.getName()); - if (write) { - builder.setWriteField(field); - } - if (read) { - builder.setReadField(field); - } - if (builders.put(field.getName(), builder) != null) { - throw new ForyJsonException("Duplicate JSON field " + field.getName()); - } - } + LinkedHashMap builders = new LinkedHashMap<>(); + addFields(type, record, writeExpose, readExpose, propertyDiscoveryEnabled, builders); + if (propertyDiscoveryEnabled && !record) { + addAccessors(type, writeExpose, readExpose, builders); } List writes = new ArrayList<>(); List reads = new ArrayList<>(); for (FieldBuilder builder : builders.values()) { + if (!builder.hasWriteSource() && !builder.hasReadSink()) { + continue; + } JsonFieldInfo field = builder.build(record); - if (builder.writeAccessor != null) { + if (builder.hasWriteSource()) { writes.add(field); } - if (builder.readField != null) { + if (builder.hasReadSink()) { reads.add(field); } } @@ -138,6 +122,76 @@ boolean record = RecordUtils.isRecord(type); type, writeArray, readArray, ObjectInstantiators.createObjectInstantiator(type)); } + private static void addFields( + Class type, + boolean record, + boolean writeExpose, + boolean readExpose, + boolean propertyDiscoveryEnabled, + LinkedHashMap builders) { + List> hierarchy = new ArrayList<>(); + for (Class current = type; + current != null && current != Object.class; + current = current.getSuperclass()) { + hierarchy.add(current); + } + for (int i = hierarchy.size() - 1; i >= 0; i--) { + Class current = hierarchy.get(i); + for (Field field : current.getDeclaredFields()) { + int modifiers = field.getModifiers(); + if (!isEligibleField(field)) { + continue; + } + boolean write = includeWrite(field, writeExpose); + boolean readAllowed = includeRead(field, readExpose); + boolean read = (record || !Modifier.isFinal(modifiers)) && readAllowed; + if (!propertyDiscoveryEnabled && !write && !read) { + continue; + } + FieldBuilder builder = + builders.computeIfAbsent(field.getName(), name -> new FieldBuilder(name)); + builder.setField(field, write, read, write, readAllowed); + } + } + } + + private static void addAccessors( + Class type, + boolean writeExpose, + boolean readExpose, + LinkedHashMap builders) { + for (Method method : type.getMethods()) { + if (!isEligibleAccessor(method)) { + continue; + } + String propertyName = getterPropertyName(method); + if (propertyName != null) { + FieldBuilder builder = builders.get(propertyName); + if (builder == null) { + if (writeExpose) { + continue; + } + builder = new FieldBuilder(propertyName); + builders.put(propertyName, builder); + } + builder.setWriteGetter(method, writeExpose); + continue; + } + propertyName = setterPropertyName(method); + if (propertyName != null) { + FieldBuilder builder = builders.get(propertyName); + if (builder == null) { + if (readExpose) { + continue; + } + builder = new FieldBuilder(propertyName); + builders.put(propertyName, builder); + } + builder.setReadSetter(method, readExpose); + } + } + } + public final Class type() { return type; } @@ -384,6 +438,53 @@ private static boolean isEligibleField(Field field) { && !field.isSynthetic(); } + private static boolean isEligibleAccessor(Method method) { + int modifiers = method.getModifiers(); + return Modifier.isPublic(modifiers) + && !Modifier.isStatic(modifiers) + && !method.isSynthetic() + && !method.isBridge(); + } + + private static String getterPropertyName(Method method) { + if (method.getParameterCount() != 0 || method.getReturnType() == void.class) { + return null; + } + String name = method.getName(); + if (name.equals("getClass")) { + return null; + } + if (name.length() > 3 && name.startsWith("get")) { + return decapitalize(name.substring(3)); + } + if (name.length() > 2 + && name.startsWith("is") + && (method.getReturnType() == boolean.class || method.getReturnType() == Boolean.class)) { + return decapitalize(name.substring(2)); + } + return null; + } + + private static String setterPropertyName(Method method) { + if (method.getParameterCount() != 1 || method.getReturnType() != void.class) { + return null; + } + String name = method.getName(); + if (name.length() > 3 && name.startsWith("set")) { + return decapitalize(name.substring(3)); + } + return null; + } + + private static String decapitalize(String name) { + if (name.length() > 1 + && Character.isUpperCase(name.charAt(0)) + && Character.isUpperCase(name.charAt(1))) { + return name; + } + return Character.toLowerCase(name.charAt(0)) + name.substring(1); + } + private static boolean includeWrite(Field field, boolean exposeMode) { return include(field, exposeMode, true); } @@ -419,8 +520,13 @@ private static Object[] recordFieldDefaults( private static final class FieldBuilder { private final String name; + private Field field; + private boolean fieldWriteAllowed; + private boolean fieldReadAllowed; private Field writeField; private Field readField; + private Method writeGetter; + private Method readSetter; private JsonFieldAccessor writeAccessor; private JsonFieldAccessor readAccessor; @@ -428,24 +534,88 @@ private FieldBuilder(String name) { this.name = name; } - private void setWriteField(Field field) { - if (writeField != null) { - throw new ForyJsonException("Duplicate public JSON field " + name); + private void setField( + Field field, + boolean writeSource, + boolean readSink, + boolean writeAllowed, + boolean readAllowed) { + if (this.field != null) { + throw new ForyJsonException("Duplicate JSON field " + name); + } + this.field = field; + fieldWriteAllowed = writeAllowed; + fieldReadAllowed = readAllowed; + if (writeSource) { + writeField = field; + } + if (readSink) { + readField = field; } - writeField = field; } - private void setReadField(Field field) { - if (readField != null) { - throw new ForyJsonException("Duplicate public JSON field " + name); + private void setWriteGetter(Method getter, boolean exposeMode) { + if (!methodAllowed(exposeMode, fieldWriteAllowed)) { + return; + } + if (writeGetter != null) { + throw new ForyJsonException("Duplicate JSON getter for property " + name); } - readField = field; + writeGetter = getter; + writeField = null; + } + + private void setReadSetter(Method setter, boolean exposeMode) { + if (!methodAllowed(exposeMode, fieldReadAllowed)) { + return; + } + if (readSetter != null) { + throw new ForyJsonException("Duplicate JSON setter for property " + name); + } + readSetter = setter; + readField = null; + } + + private boolean hasWriteSource() { + return writeGetter != null || writeField != null; + } + + private boolean hasReadSink() { + return readSetter != null || readField != null; } private JsonFieldInfo build(boolean record) { - writeAccessor = writeField == null ? null : JsonFieldAccessor.forField(writeField); - readAccessor = readField == null || record ? null : JsonFieldAccessor.forField(readField); - return new JsonFieldInfo(name, writeField, readField, writeAccessor, readAccessor); + validateTypes(); + writeAccessor = + writeGetter != null + ? JsonFieldAccessor.forGetter(writeGetter) + : (writeField == null ? null : JsonFieldAccessor.forField(writeField)); + readAccessor = + readSetter != null + ? JsonFieldAccessor.forSetter(readSetter) + : (readField == null || record ? null : JsonFieldAccessor.forField(readField)); + return new JsonFieldInfo( + name, writeField, writeGetter, readField, readSetter, writeAccessor, readAccessor); + } + + private boolean methodAllowed(boolean exposeMode, boolean fieldAllowed) { + // A same-named field owns @Expose/@JsonIgnore direction decisions for the JSON property. + return field == null ? !exposeMode : fieldAllowed; + } + + private void validateTypes() { + Type writeType = + writeGetter == null ? fieldType(writeField) : writeGetter.getGenericReturnType(); + Type readType = + readSetter == null ? fieldType(readField) : readSetter.getGenericParameterTypes()[0]; + if (writeType != null && readType != null && !writeType.equals(readType)) { + throw new ForyJsonException( + "Conflicting JSON property types for " + name + ": " + writeType + " and " + readType); + } + } + + private static Type fieldType(Field field) { + return field == null ? null : field.getGenericType(); } } } diff --git a/java/fory-json/src/main/java/org/apache/fory/json/codec/CollectionCodec.java b/java/fory-json/src/main/java/org/apache/fory/json/codec/CollectionCodec.java index dfd70c8346..698551d5ab 100644 --- a/java/fory-json/src/main/java/org/apache/fory/json/codec/CollectionCodec.java +++ b/java/fory-json/src/main/java/org/apache/fory/json/codec/CollectionCodec.java @@ -50,10 +50,12 @@ public abstract class CollectionCodec extends AbstractJsonCodec { private final TypeRef typeRef; private final CollectionFactory factory; + private final boolean createsArrayList; CollectionCodec(TypeRef typeRef, CollectionFactory factory) { this.typeRef = typeRef; this.factory = factory; + this.createsArrayList = factory.createsArrayList(); } public static CollectionCodec create( @@ -118,6 +120,10 @@ final Collection newCollection() { return factory.newCollection(); } + final boolean createsArrayList() { + return createsArrayList; + } + public abstract Object readLatin1NonNull( Latin1JsonReader reader, JsonTypeInfo typeInfo, JsonTypeResolver resolver); @@ -166,7 +172,7 @@ private static CollectionFactory collectionFactory(Class rawType, Class el if (Queue.class.isAssignableFrom(rawType)) { return ArrayDeque::new; } - return () -> new ArrayList<>(0); + return CollectionFactory.ARRAY_LIST; } return () -> { try { @@ -178,7 +184,24 @@ private static CollectionFactory collectionFactory(Class rawType, Class el } private interface CollectionFactory { + CollectionFactory ARRAY_LIST = + new CollectionFactory() { + @Override + public Collection newCollection() { + return new ArrayList<>(0); + } + + @Override + public boolean createsArrayList() { + return true; + } + }; + Collection newCollection(); + + default boolean createsArrayList() { + return false; + } } public abstract static class DirectCollectionCodec extends CollectionCodec { @@ -212,6 +235,9 @@ public final Object readLatin1( @Override public final Object readLatin1NonNull( Latin1JsonReader reader, JsonTypeInfo typeInfo, JsonTypeResolver resolver) { + if (createsArrayList()) { + return readLatin1ArrayListNonNull(reader); + } reader.enterDepth(); Collection collection = newCollection(); reader.expectNextToken('['); @@ -236,6 +262,9 @@ public final Object readUtf16( @Override public final Object readUtf16NonNull( Utf16JsonReader reader, JsonTypeInfo typeInfo, JsonTypeResolver resolver) { + if (createsArrayList()) { + return readUtf16ArrayListNonNull(reader); + } reader.enterDepth(); Collection collection = newCollection(); reader.expectNextToken('['); @@ -260,6 +289,9 @@ public final Object readUtf8( @Override public final Object readUtf8NonNull( Utf8JsonReader reader, JsonTypeInfo typeInfo, JsonTypeResolver resolver) { + if (createsArrayList()) { + return readUtf8ArrayListNonNull(reader); + } reader.enterDepth(); Collection collection = newCollection(); reader.expectNextToken('['); @@ -272,6 +304,357 @@ public final Object readUtf8NonNull( return collection; } + private ArrayList readLatin1ArrayListNonNull(Latin1JsonReader reader) { + reader.enterDepth(); + reader.expectNextToken('['); + if (reader.consumeNextToken(']')) { + reader.exitDepth(); + return new ArrayList<>(0); + } + Object e0 = readNullableLatin1Element(reader); + if (!reader.consumeNextCommaOrEndArray()) { + reader.exitDepth(); + ArrayList list = new ArrayList<>(1); + list.add(e0); + return list; + } + Object e1 = readNullableLatin1Element(reader); + if (!reader.consumeNextCommaOrEndArray()) { + reader.exitDepth(); + ArrayList list = new ArrayList<>(2); + list.add(e0); + list.add(e1); + return list; + } + Object e2 = readNullableLatin1Element(reader); + if (!reader.consumeNextCommaOrEndArray()) { + reader.exitDepth(); + ArrayList list = new ArrayList<>(3); + list.add(e0); + list.add(e1); + list.add(e2); + return list; + } + Object e3 = readNullableLatin1Element(reader); + if (!reader.consumeNextCommaOrEndArray()) { + reader.exitDepth(); + ArrayList list = new ArrayList<>(4); + list.add(e0); + list.add(e1); + list.add(e2); + list.add(e3); + return list; + } + return readLatin1ArrayListTail(reader, e0, e1, e2, e3); + } + + private ArrayList readLatin1ArrayListTail( + Latin1JsonReader reader, Object e0, Object e1, Object e2, Object e3) { + Object e4 = readNullableLatin1Element(reader); + if (!reader.consumeNextCommaOrEndArray()) { + reader.exitDepth(); + ArrayList list = new ArrayList<>(5); + list.add(e0); + list.add(e1); + list.add(e2); + list.add(e3); + list.add(e4); + return list; + } + Object e5 = readNullableLatin1Element(reader); + if (!reader.consumeNextCommaOrEndArray()) { + reader.exitDepth(); + ArrayList list = new ArrayList<>(6); + list.add(e0); + list.add(e1); + list.add(e2); + list.add(e3); + list.add(e4); + list.add(e5); + return list; + } + return readLatin1ArrayListLongTail(reader, e0, e1, e2, e3, e4, e5); + } + + private ArrayList readLatin1ArrayListLongTail( + Latin1JsonReader reader, Object e0, Object e1, Object e2, Object e3, Object e4, Object e5) { + Object e6 = readNullableLatin1Element(reader); + if (!reader.consumeNextCommaOrEndArray()) { + reader.exitDepth(); + ArrayList list = new ArrayList<>(7); + list.add(e0); + list.add(e1); + list.add(e2); + list.add(e3); + list.add(e4); + list.add(e5); + list.add(e6); + return list; + } + Object e7 = readNullableLatin1Element(reader); + if (!reader.consumeNextCommaOrEndArray()) { + reader.exitDepth(); + ArrayList list = new ArrayList<>(8); + list.add(e0); + list.add(e1); + list.add(e2); + list.add(e3); + list.add(e4); + list.add(e5); + list.add(e6); + list.add(e7); + return list; + } + ArrayList list = new ArrayList<>(9); + list.add(e0); + list.add(e1); + list.add(e2); + list.add(e3); + list.add(e4); + list.add(e5); + list.add(e6); + list.add(e7); + do { + list.add(readNullableLatin1Element(reader)); + } while (reader.consumeNextCommaOrEndArray()); + reader.exitDepth(); + return list; + } + + private ArrayList readUtf16ArrayListNonNull(Utf16JsonReader reader) { + reader.enterDepth(); + reader.expectNextToken('['); + if (reader.consumeNextToken(']')) { + reader.exitDepth(); + return new ArrayList<>(0); + } + Object e0 = readNullableUtf16Element(reader); + if (!reader.consumeNextCommaOrEndArray()) { + reader.exitDepth(); + ArrayList list = new ArrayList<>(1); + list.add(e0); + return list; + } + Object e1 = readNullableUtf16Element(reader); + if (!reader.consumeNextCommaOrEndArray()) { + reader.exitDepth(); + ArrayList list = new ArrayList<>(2); + list.add(e0); + list.add(e1); + return list; + } + Object e2 = readNullableUtf16Element(reader); + if (!reader.consumeNextCommaOrEndArray()) { + reader.exitDepth(); + ArrayList list = new ArrayList<>(3); + list.add(e0); + list.add(e1); + list.add(e2); + return list; + } + Object e3 = readNullableUtf16Element(reader); + if (!reader.consumeNextCommaOrEndArray()) { + reader.exitDepth(); + ArrayList list = new ArrayList<>(4); + list.add(e0); + list.add(e1); + list.add(e2); + list.add(e3); + return list; + } + return readUtf16ArrayListTail(reader, e0, e1, e2, e3); + } + + private ArrayList readUtf16ArrayListTail( + Utf16JsonReader reader, Object e0, Object e1, Object e2, Object e3) { + Object e4 = readNullableUtf16Element(reader); + if (!reader.consumeNextCommaOrEndArray()) { + reader.exitDepth(); + ArrayList list = new ArrayList<>(5); + list.add(e0); + list.add(e1); + list.add(e2); + list.add(e3); + list.add(e4); + return list; + } + Object e5 = readNullableUtf16Element(reader); + if (!reader.consumeNextCommaOrEndArray()) { + reader.exitDepth(); + ArrayList list = new ArrayList<>(6); + list.add(e0); + list.add(e1); + list.add(e2); + list.add(e3); + list.add(e4); + list.add(e5); + return list; + } + return readUtf16ArrayListLongTail(reader, e0, e1, e2, e3, e4, e5); + } + + private ArrayList readUtf16ArrayListLongTail( + Utf16JsonReader reader, Object e0, Object e1, Object e2, Object e3, Object e4, Object e5) { + Object e6 = readNullableUtf16Element(reader); + if (!reader.consumeNextCommaOrEndArray()) { + reader.exitDepth(); + ArrayList list = new ArrayList<>(7); + list.add(e0); + list.add(e1); + list.add(e2); + list.add(e3); + list.add(e4); + list.add(e5); + list.add(e6); + return list; + } + Object e7 = readNullableUtf16Element(reader); + if (!reader.consumeNextCommaOrEndArray()) { + reader.exitDepth(); + ArrayList list = new ArrayList<>(8); + list.add(e0); + list.add(e1); + list.add(e2); + list.add(e3); + list.add(e4); + list.add(e5); + list.add(e6); + list.add(e7); + return list; + } + ArrayList list = new ArrayList<>(9); + list.add(e0); + list.add(e1); + list.add(e2); + list.add(e3); + list.add(e4); + list.add(e5); + list.add(e6); + list.add(e7); + do { + list.add(readNullableUtf16Element(reader)); + } while (reader.consumeNextCommaOrEndArray()); + reader.exitDepth(); + return list; + } + + private ArrayList readUtf8ArrayListNonNull(Utf8JsonReader reader) { + reader.enterDepth(); + reader.expectNextToken('['); + if (reader.consumeNextToken(']')) { + reader.exitDepth(); + return new ArrayList<>(0); + } + Object e0 = readNullableUtf8Element(reader); + if (!reader.consumeNextCommaOrEndArray()) { + reader.exitDepth(); + ArrayList list = new ArrayList<>(1); + list.add(e0); + return list; + } + Object e1 = readNullableUtf8Element(reader); + if (!reader.consumeNextCommaOrEndArray()) { + reader.exitDepth(); + ArrayList list = new ArrayList<>(2); + list.add(e0); + list.add(e1); + return list; + } + Object e2 = readNullableUtf8Element(reader); + if (!reader.consumeNextCommaOrEndArray()) { + reader.exitDepth(); + ArrayList list = new ArrayList<>(3); + list.add(e0); + list.add(e1); + list.add(e2); + return list; + } + Object e3 = readNullableUtf8Element(reader); + if (!reader.consumeNextCommaOrEndArray()) { + reader.exitDepth(); + ArrayList list = new ArrayList<>(4); + list.add(e0); + list.add(e1); + list.add(e2); + list.add(e3); + return list; + } + return readUtf8ArrayListTail(reader, e0, e1, e2, e3); + } + + private ArrayList readUtf8ArrayListTail( + Utf8JsonReader reader, Object e0, Object e1, Object e2, Object e3) { + Object e4 = readNullableUtf8Element(reader); + if (!reader.consumeNextCommaOrEndArray()) { + reader.exitDepth(); + ArrayList list = new ArrayList<>(5); + list.add(e0); + list.add(e1); + list.add(e2); + list.add(e3); + list.add(e4); + return list; + } + Object e5 = readNullableUtf8Element(reader); + if (!reader.consumeNextCommaOrEndArray()) { + reader.exitDepth(); + ArrayList list = new ArrayList<>(6); + list.add(e0); + list.add(e1); + list.add(e2); + list.add(e3); + list.add(e4); + list.add(e5); + return list; + } + return readUtf8ArrayListLongTail(reader, e0, e1, e2, e3, e4, e5); + } + + private ArrayList readUtf8ArrayListLongTail( + Utf8JsonReader reader, Object e0, Object e1, Object e2, Object e3, Object e4, Object e5) { + Object e6 = readNullableUtf8Element(reader); + if (!reader.consumeNextCommaOrEndArray()) { + reader.exitDepth(); + ArrayList list = new ArrayList<>(7); + list.add(e0); + list.add(e1); + list.add(e2); + list.add(e3); + list.add(e4); + list.add(e5); + list.add(e6); + return list; + } + Object e7 = readNullableUtf8Element(reader); + if (!reader.consumeNextCommaOrEndArray()) { + reader.exitDepth(); + ArrayList list = new ArrayList<>(8); + list.add(e0); + list.add(e1); + list.add(e2); + list.add(e3); + list.add(e4); + list.add(e5); + list.add(e6); + list.add(e7); + return list; + } + ArrayList list = new ArrayList<>(9); + list.add(e0); + list.add(e1); + list.add(e2); + list.add(e3); + list.add(e4); + list.add(e5); + list.add(e6); + list.add(e7); + do { + list.add(readNullableUtf8Element(reader)); + } while (reader.consumeNextCommaOrEndArray()); + reader.exitDepth(); + return list; + } + abstract Object readElement(JsonReader reader); Object readNullableElement(JsonReader reader) { @@ -447,13 +830,26 @@ private ObjectCollectionCodec( @Override void writeNonNull(JsonWriter writer, Object value, JsonTypeResolver resolver) { writer.writeArrayStart(); - int index = 0; - for (Object element : (Collection) value) { - writer.writeComma(index++); - if (element == null) { - writer.writeNull(); - } else { - elementCodec.writeNonNull(writer, element, resolver); + if (value.getClass() == ArrayList.class) { + ArrayList list = (ArrayList) value; + for (int index = 0, size = list.size(); index < size; index++) { + Object element = list.get(index); + writer.writeComma(index); + if (element == null) { + writer.writeNull(); + } else { + elementCodec.writeNonNull(writer, element, resolver); + } + } + } else { + int index = 0; + for (Object element : (Collection) value) { + writer.writeComma(index++); + if (element == null) { + writer.writeNull(); + } else { + elementCodec.writeNonNull(writer, element, resolver); + } } } writer.writeArrayEnd(); @@ -462,13 +858,26 @@ void writeNonNull(JsonWriter writer, Object value, JsonTypeResolver resolver) { @Override void writeStringNonNull(StringJsonWriter writer, Object value, JsonTypeResolver resolver) { writer.writeArrayStart(); - int index = 0; - for (Object element : (Collection) value) { - writer.writeComma(index++); - if (element == null) { - writer.writeNull(); - } else { - elementCodec.writeStringNonNull(writer, element, resolver); + if (value.getClass() == ArrayList.class) { + ArrayList list = (ArrayList) value; + for (int index = 0, size = list.size(); index < size; index++) { + Object element = list.get(index); + writer.writeComma(index); + if (element == null) { + writer.writeNull(); + } else { + elementCodec.writeStringNonNull(writer, element, resolver); + } + } + } else { + int index = 0; + for (Object element : (Collection) value) { + writer.writeComma(index++); + if (element == null) { + writer.writeNull(); + } else { + elementCodec.writeStringNonNull(writer, element, resolver); + } } } writer.writeArrayEnd(); @@ -477,13 +886,26 @@ void writeStringNonNull(StringJsonWriter writer, Object value, JsonTypeResolver @Override void writeUtf8NonNull(Utf8JsonWriter writer, Object value, JsonTypeResolver resolver) { writer.writeArrayStart(); - int index = 0; - for (Object element : (Collection) value) { - writer.writeComma(index++); - if (element == null) { - writer.writeNull(); - } else { - elementCodec.writeUtf8NonNull(writer, element, resolver); + if (value.getClass() == ArrayList.class) { + ArrayList list = (ArrayList) value; + for (int index = 0, size = list.size(); index < size; index++) { + Object element = list.get(index); + writer.writeComma(index); + if (element == null) { + writer.writeNull(); + } else { + elementCodec.writeUtf8NonNull(writer, element, resolver); + } + } + } else { + int index = 0; + for (Object element : (Collection) value) { + writer.writeComma(index++); + if (element == null) { + writer.writeNull(); + } else { + elementCodec.writeUtf8NonNull(writer, element, resolver); + } } } writer.writeArrayEnd(); @@ -518,6 +940,9 @@ public Object readLatin1( @Override public Object readLatin1NonNull( Latin1JsonReader reader, JsonTypeInfo typeInfo, JsonTypeResolver resolver) { + if (createsArrayList()) { + return readLatin1ArrayListNonNull(reader, resolver); + } reader.enterDepth(); Collection collection = newCollection(); reader.expectNextToken('['); @@ -545,6 +970,9 @@ public Object readUtf16( @Override public Object readUtf16NonNull( Utf16JsonReader reader, JsonTypeInfo typeInfo, JsonTypeResolver resolver) { + if (createsArrayList()) { + return readUtf16ArrayListNonNull(reader, resolver); + } reader.enterDepth(); Collection collection = newCollection(); reader.expectNextToken('['); @@ -572,6 +1000,9 @@ public Object readUtf8( @Override public Object readUtf8NonNull( Utf8JsonReader reader, JsonTypeInfo typeInfo, JsonTypeResolver resolver) { + if (createsArrayList()) { + return readUtf8ArrayListNonNull(reader, resolver); + } reader.enterDepth(); Collection collection = newCollection(); reader.expectNextToken('['); @@ -586,6 +1017,414 @@ public Object readUtf8NonNull( reader.exitDepth(); return collection; } + + private ArrayList readLatin1ArrayListNonNull( + Latin1JsonReader reader, JsonTypeResolver resolver) { + reader.enterDepth(); + reader.expectNextToken('['); + if (reader.consumeNextToken(']')) { + reader.exitDepth(); + return new ArrayList<>(0); + } + Object e0 = readNullableLatin1Element(reader, resolver); + if (!reader.consumeNextCommaOrEndArray()) { + reader.exitDepth(); + ArrayList list = new ArrayList<>(1); + list.add(e0); + return list; + } + Object e1 = readNullableLatin1Element(reader, resolver); + if (!reader.consumeNextCommaOrEndArray()) { + reader.exitDepth(); + ArrayList list = new ArrayList<>(2); + list.add(e0); + list.add(e1); + return list; + } + Object e2 = readNullableLatin1Element(reader, resolver); + if (!reader.consumeNextCommaOrEndArray()) { + reader.exitDepth(); + ArrayList list = new ArrayList<>(3); + list.add(e0); + list.add(e1); + list.add(e2); + return list; + } + Object e3 = readNullableLatin1Element(reader, resolver); + if (!reader.consumeNextCommaOrEndArray()) { + reader.exitDepth(); + ArrayList list = new ArrayList<>(4); + list.add(e0); + list.add(e1); + list.add(e2); + list.add(e3); + return list; + } + return readLatin1ArrayListTail(reader, resolver, e0, e1, e2, e3); + } + + private ArrayList readLatin1ArrayListTail( + Latin1JsonReader reader, + JsonTypeResolver resolver, + Object e0, + Object e1, + Object e2, + Object e3) { + Object e4 = readNullableLatin1Element(reader, resolver); + if (!reader.consumeNextCommaOrEndArray()) { + reader.exitDepth(); + ArrayList list = new ArrayList<>(5); + list.add(e0); + list.add(e1); + list.add(e2); + list.add(e3); + list.add(e4); + return list; + } + Object e5 = readNullableLatin1Element(reader, resolver); + if (!reader.consumeNextCommaOrEndArray()) { + reader.exitDepth(); + ArrayList list = new ArrayList<>(6); + list.add(e0); + list.add(e1); + list.add(e2); + list.add(e3); + list.add(e4); + list.add(e5); + return list; + } + return readLatin1ArrayListLongTail(reader, resolver, e0, e1, e2, e3, e4, e5); + } + + private ArrayList readLatin1ArrayListLongTail( + Latin1JsonReader reader, + JsonTypeResolver resolver, + Object e0, + Object e1, + Object e2, + Object e3, + Object e4, + Object e5) { + Object e6 = readNullableLatin1Element(reader, resolver); + if (!reader.consumeNextCommaOrEndArray()) { + reader.exitDepth(); + ArrayList list = new ArrayList<>(7); + list.add(e0); + list.add(e1); + list.add(e2); + list.add(e3); + list.add(e4); + list.add(e5); + list.add(e6); + return list; + } + Object e7 = readNullableLatin1Element(reader, resolver); + if (!reader.consumeNextCommaOrEndArray()) { + reader.exitDepth(); + ArrayList list = new ArrayList<>(8); + list.add(e0); + list.add(e1); + list.add(e2); + list.add(e3); + list.add(e4); + list.add(e5); + list.add(e6); + list.add(e7); + return list; + } + ArrayList list = new ArrayList<>(9); + list.add(e0); + list.add(e1); + list.add(e2); + list.add(e3); + list.add(e4); + list.add(e5); + list.add(e6); + list.add(e7); + do { + list.add(readNullableLatin1Element(reader, resolver)); + } while (reader.consumeNextCommaOrEndArray()); + reader.exitDepth(); + return list; + } + + private ArrayList readUtf16ArrayListNonNull( + Utf16JsonReader reader, JsonTypeResolver resolver) { + reader.enterDepth(); + reader.expectNextToken('['); + if (reader.consumeNextToken(']')) { + reader.exitDepth(); + return new ArrayList<>(0); + } + Object e0 = readNullableUtf16Element(reader, resolver); + if (!reader.consumeNextCommaOrEndArray()) { + reader.exitDepth(); + ArrayList list = new ArrayList<>(1); + list.add(e0); + return list; + } + Object e1 = readNullableUtf16Element(reader, resolver); + if (!reader.consumeNextCommaOrEndArray()) { + reader.exitDepth(); + ArrayList list = new ArrayList<>(2); + list.add(e0); + list.add(e1); + return list; + } + Object e2 = readNullableUtf16Element(reader, resolver); + if (!reader.consumeNextCommaOrEndArray()) { + reader.exitDepth(); + ArrayList list = new ArrayList<>(3); + list.add(e0); + list.add(e1); + list.add(e2); + return list; + } + Object e3 = readNullableUtf16Element(reader, resolver); + if (!reader.consumeNextCommaOrEndArray()) { + reader.exitDepth(); + ArrayList list = new ArrayList<>(4); + list.add(e0); + list.add(e1); + list.add(e2); + list.add(e3); + return list; + } + return readUtf16ArrayListTail(reader, resolver, e0, e1, e2, e3); + } + + private ArrayList readUtf16ArrayListTail( + Utf16JsonReader reader, + JsonTypeResolver resolver, + Object e0, + Object e1, + Object e2, + Object e3) { + Object e4 = readNullableUtf16Element(reader, resolver); + if (!reader.consumeNextCommaOrEndArray()) { + reader.exitDepth(); + ArrayList list = new ArrayList<>(5); + list.add(e0); + list.add(e1); + list.add(e2); + list.add(e3); + list.add(e4); + return list; + } + Object e5 = readNullableUtf16Element(reader, resolver); + if (!reader.consumeNextCommaOrEndArray()) { + reader.exitDepth(); + ArrayList list = new ArrayList<>(6); + list.add(e0); + list.add(e1); + list.add(e2); + list.add(e3); + list.add(e4); + list.add(e5); + return list; + } + return readUtf16ArrayListLongTail(reader, resolver, e0, e1, e2, e3, e4, e5); + } + + private ArrayList readUtf16ArrayListLongTail( + Utf16JsonReader reader, + JsonTypeResolver resolver, + Object e0, + Object e1, + Object e2, + Object e3, + Object e4, + Object e5) { + Object e6 = readNullableUtf16Element(reader, resolver); + if (!reader.consumeNextCommaOrEndArray()) { + reader.exitDepth(); + ArrayList list = new ArrayList<>(7); + list.add(e0); + list.add(e1); + list.add(e2); + list.add(e3); + list.add(e4); + list.add(e5); + list.add(e6); + return list; + } + Object e7 = readNullableUtf16Element(reader, resolver); + if (!reader.consumeNextCommaOrEndArray()) { + reader.exitDepth(); + ArrayList list = new ArrayList<>(8); + list.add(e0); + list.add(e1); + list.add(e2); + list.add(e3); + list.add(e4); + list.add(e5); + list.add(e6); + list.add(e7); + return list; + } + ArrayList list = new ArrayList<>(9); + list.add(e0); + list.add(e1); + list.add(e2); + list.add(e3); + list.add(e4); + list.add(e5); + list.add(e6); + list.add(e7); + do { + list.add(readNullableUtf16Element(reader, resolver)); + } while (reader.consumeNextCommaOrEndArray()); + reader.exitDepth(); + return list; + } + + private ArrayList readUtf8ArrayListNonNull( + Utf8JsonReader reader, JsonTypeResolver resolver) { + reader.enterDepth(); + reader.expectNextToken('['); + if (reader.consumeNextToken(']')) { + reader.exitDepth(); + return new ArrayList<>(0); + } + Object e0 = readNullableUtf8Element(reader, resolver); + if (!reader.consumeNextCommaOrEndArray()) { + reader.exitDepth(); + ArrayList list = new ArrayList<>(1); + list.add(e0); + return list; + } + Object e1 = readNullableUtf8Element(reader, resolver); + if (!reader.consumeNextCommaOrEndArray()) { + reader.exitDepth(); + ArrayList list = new ArrayList<>(2); + list.add(e0); + list.add(e1); + return list; + } + Object e2 = readNullableUtf8Element(reader, resolver); + if (!reader.consumeNextCommaOrEndArray()) { + reader.exitDepth(); + ArrayList list = new ArrayList<>(3); + list.add(e0); + list.add(e1); + list.add(e2); + return list; + } + Object e3 = readNullableUtf8Element(reader, resolver); + if (!reader.consumeNextCommaOrEndArray()) { + reader.exitDepth(); + ArrayList list = new ArrayList<>(4); + list.add(e0); + list.add(e1); + list.add(e2); + list.add(e3); + return list; + } + return readUtf8ArrayListTail(reader, resolver, e0, e1, e2, e3); + } + + private ArrayList readUtf8ArrayListTail( + Utf8JsonReader reader, + JsonTypeResolver resolver, + Object e0, + Object e1, + Object e2, + Object e3) { + Object e4 = readNullableUtf8Element(reader, resolver); + if (!reader.consumeNextCommaOrEndArray()) { + reader.exitDepth(); + ArrayList list = new ArrayList<>(5); + list.add(e0); + list.add(e1); + list.add(e2); + list.add(e3); + list.add(e4); + return list; + } + Object e5 = readNullableUtf8Element(reader, resolver); + if (!reader.consumeNextCommaOrEndArray()) { + reader.exitDepth(); + ArrayList list = new ArrayList<>(6); + list.add(e0); + list.add(e1); + list.add(e2); + list.add(e3); + list.add(e4); + list.add(e5); + return list; + } + return readUtf8ArrayListLongTail(reader, resolver, e0, e1, e2, e3, e4, e5); + } + + private ArrayList readUtf8ArrayListLongTail( + Utf8JsonReader reader, + JsonTypeResolver resolver, + Object e0, + Object e1, + Object e2, + Object e3, + Object e4, + Object e5) { + Object e6 = readNullableUtf8Element(reader, resolver); + if (!reader.consumeNextCommaOrEndArray()) { + reader.exitDepth(); + ArrayList list = new ArrayList<>(7); + list.add(e0); + list.add(e1); + list.add(e2); + list.add(e3); + list.add(e4); + list.add(e5); + list.add(e6); + return list; + } + Object e7 = readNullableUtf8Element(reader, resolver); + if (!reader.consumeNextCommaOrEndArray()) { + reader.exitDepth(); + ArrayList list = new ArrayList<>(8); + list.add(e0); + list.add(e1); + list.add(e2); + list.add(e3); + list.add(e4); + list.add(e5); + list.add(e6); + list.add(e7); + return list; + } + ArrayList list = new ArrayList<>(9); + list.add(e0); + list.add(e1); + list.add(e2); + list.add(e3); + list.add(e4); + list.add(e5); + list.add(e6); + list.add(e7); + do { + list.add(readNullableUtf8Element(reader, resolver)); + } while (reader.consumeNextCommaOrEndArray()); + reader.exitDepth(); + return list; + } + + private Object readNullableLatin1Element(Latin1JsonReader reader, JsonTypeResolver resolver) { + return reader.tryReadNextNullToken() + ? null + : elementCodec.readLatin1NonNull(reader, elementTypeInfo, resolver); + } + + private Object readNullableUtf16Element(Utf16JsonReader reader, JsonTypeResolver resolver) { + return reader.tryReadNextNullToken() + ? null + : elementCodec.readUtf16NonNull(reader, elementTypeInfo, resolver); + } + + private Object readNullableUtf8Element(Utf8JsonReader reader, JsonTypeResolver resolver) { + return reader.tryReadNextNullToken() + ? null + : elementCodec.readUtf8NonNull(reader, elementTypeInfo, resolver); + } } public static final class StringCollectionCodec extends DirectCollectionCodec { diff --git a/java/fory-json/src/main/java/org/apache/fory/json/codec/ScalarCodecs.java b/java/fory-json/src/main/java/org/apache/fory/json/codec/ScalarCodecs.java index 2b58fc72c2..2380a4c933 100644 --- a/java/fory-json/src/main/java/org/apache/fory/json/codec/ScalarCodecs.java +++ b/java/fory-json/src/main/java/org/apache/fory/json/codec/ScalarCodecs.java @@ -56,8 +56,11 @@ import java.util.UUID; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicIntegerArray; import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicLongArray; import java.util.concurrent.atomic.AtomicReference; +import java.util.concurrent.atomic.AtomicReferenceArray; import java.util.regex.Pattern; import org.apache.fory.json.ForyJsonException; import org.apache.fory.json.meta.JsonAsciiToken; @@ -438,7 +441,7 @@ void writeUtf8NonNull(Utf8JsonWriter writer, Object value, JsonTypeResolver reso @Override Object readNonNull(JsonReader reader, JsonTypeInfo typeInfo, JsonTypeResolver resolver) { - return Double.parseDouble(reader.readNumber()); + return reader.readDouble(); } @Override @@ -451,9 +454,9 @@ public void readField( if (reader.peekNull()) { readFieldDefault(reader, object, accessor, typeInfo, resolver); } else if (typeInfo.primitive()) { - accessor.putDouble(object, Double.parseDouble(reader.readNumber())); + accessor.putDouble(object, reader.readDouble()); } else { - accessor.putObject(object, Double.parseDouble(reader.readNumber())); + accessor.putObject(object, reader.readDouble()); } } } @@ -509,13 +512,16 @@ final void writeNonNull(JsonWriter writer, Object value, JsonTypeResolver resolv } @Override - final void writeUtf8NonNull(Utf8JsonWriter writer, Object value, JsonTypeResolver resolver) { + void writeUtf8NonNull(Utf8JsonWriter writer, Object value, JsonTypeResolver resolver) { writer.writeString(toJsonString(value)); } @Override final Object readNonNull(JsonReader reader, JsonTypeInfo typeInfo, JsonTypeResolver resolver) { - String value = reader.readString(); + return readStringValue(reader.readString(), typeInfo); + } + + final Object readStringValue(String value, JsonTypeInfo typeInfo) { try { return fromJsonString(value); } catch (ForyJsonException e) { @@ -578,6 +584,24 @@ String toJsonNumber(Object value) { Object fromJsonNumber(String value) { return new BigDecimal(value); } + + @Override + public Object readUtf8( + Utf8JsonReader reader, JsonTypeInfo typeInfo, JsonTypeResolver resolver) { + if (reader.tryReadNullToken()) { + return null; + } + return reader.readBigDecimal(); + } + + @Override + public Object readLatin1( + Latin1JsonReader reader, JsonTypeInfo typeInfo, JsonTypeResolver resolver) { + if (reader.tryReadNullToken()) { + return null; + } + return reader.readBigDecimal(); + } } public static final class Float16Codec extends AbstractJsonCodec { @@ -742,10 +766,45 @@ String toJsonString(Object value) { return value.toString(); } + @Override + void writeUtf8NonNull(Utf8JsonWriter writer, Object value, JsonTypeResolver resolver) { + writer.writeUuid((UUID) value); + } + @Override Object fromJsonString(String value) { return UUID.fromString(value); } + + @Override + public Object readUtf8( + Utf8JsonReader reader, JsonTypeInfo typeInfo, JsonTypeResolver resolver) { + if (reader.tryReadNullToken()) { + return null; + } + try { + return reader.readUuid(); + } catch (ForyJsonException e) { + throw e; + } catch (RuntimeException e) { + throw new ForyJsonException("Invalid " + typeInfo.rawType().getName() + " JSON string", e); + } + } + + @Override + public Object readLatin1( + Latin1JsonReader reader, JsonTypeInfo typeInfo, JsonTypeResolver resolver) { + if (reader.tryReadNullToken()) { + return null; + } + try { + return reader.readUuid(); + } catch (ForyJsonException e) { + throw e; + } catch (RuntimeException e) { + throw new ForyJsonException("Invalid " + typeInfo.rawType().getName() + " JSON string", e); + } + } } public static final class LocaleCodec extends StringValueCodec { @@ -895,12 +954,53 @@ String toJsonString(Object value) { return value.toString(); } + @Override + void writeUtf8NonNull(Utf8JsonWriter writer, Object value, JsonTypeResolver resolver) { + writer.writeLocalDate((LocalDate) value); + } + @Override Object fromJsonString(String value) { - if (value.length() > 10 && value.charAt(10) == 'T') { - return LocalDate.parse(value.substring(0, 10)); + return parseIsoLocalDate(value); + } + + @Override + public Object readUtf8( + Utf8JsonReader reader, JsonTypeInfo typeInfo, JsonTypeResolver resolver) { + if (reader.tryReadNullToken()) { + return null; + } + try { + return reader.readIsoLocalDate(); + } catch (RuntimeException e) { + return readStringValue(reader.readString(), typeInfo); + } + } + + @Override + public Object readLatin1( + Latin1JsonReader reader, JsonTypeInfo typeInfo, JsonTypeResolver resolver) { + if (reader.tryReadNullToken()) { + return null; + } + try { + return reader.readIsoLocalDate(); + } catch (RuntimeException e) { + return readStringValue(reader.readString(), typeInfo); + } + } + + @Override + public Object readUtf16( + Utf16JsonReader reader, JsonTypeInfo typeInfo, JsonTypeResolver resolver) { + if (reader.tryReadNullToken()) { + return null; + } + try { + return reader.readIsoLocalDate(); + } catch (RuntimeException e) { + return readStringValue(reader.readString(), typeInfo); } - return LocalDate.parse(value); } } @@ -1080,12 +1180,179 @@ String toJsonString(Object value) { return value.toString(); } + @Override + void writeUtf8NonNull(Utf8JsonWriter writer, Object value, JsonTypeResolver resolver) { + writer.writeOffsetDateTime((OffsetDateTime) value); + } + @Override Object fromJsonString(String value) { + return parseIsoOffsetDateTime(value); + } + + @Override + public Object readUtf8( + Utf8JsonReader reader, JsonTypeInfo typeInfo, JsonTypeResolver resolver) { + if (reader.tryReadNullToken()) { + return null; + } + try { + return reader.readIsoOffsetDateTime(); + } catch (RuntimeException e) { + return readStringValue(reader.readString(), typeInfo); + } + } + + @Override + public Object readLatin1( + Latin1JsonReader reader, JsonTypeInfo typeInfo, JsonTypeResolver resolver) { + if (reader.tryReadNullToken()) { + return null; + } + try { + return reader.readIsoOffsetDateTime(); + } catch (RuntimeException e) { + return readStringValue(reader.readString(), typeInfo); + } + } + + @Override + public Object readUtf16( + Utf16JsonReader reader, JsonTypeInfo typeInfo, JsonTypeResolver resolver) { + if (reader.tryReadNullToken()) { + return null; + } + try { + return reader.readIsoOffsetDateTime(); + } catch (RuntimeException e) { + return readStringValue(reader.readString(), typeInfo); + } + } + } + + private static LocalDate parseIsoLocalDate(String value) { + int length = value.length(); + if (length >= 10 + && (length == 10 || value.charAt(10) == 'T') + && value.charAt(4) == '-' + && value.charAt(7) == '-') { + try { + return LocalDate.of(parse4(value, 0), parse2(value, 5), parse2(value, 8)); + } catch (RuntimeException e) { + if (length > 10 && value.charAt(10) == 'T') { + return LocalDate.parse(value.substring(0, 10)); + } + return LocalDate.parse(value); + } + } + return LocalDate.parse(value); + } + + private static OffsetDateTime parseIsoOffsetDateTime(String value) { + try { + // java.time emits these ISO forms, and parsing them directly avoids DateTimeFormatter's + // parsed-field maps on JSON scalar hot paths. Uncommon forms still use the JDK parser. + return parseIsoOffsetDateTimeFast(value); + } catch (RuntimeException e) { return OffsetDateTime.parse(value); } } + private static OffsetDateTime parseIsoOffsetDateTimeFast(String value) { + int length = value.length(); + if (length < 17 + || value.charAt(4) != '-' + || value.charAt(7) != '-' + || value.charAt(10) != 'T' + || value.charAt(13) != ':') { + throw new IllegalArgumentException(); + } + int year = parse4(value, 0); + int month = parse2(value, 5); + int day = parse2(value, 8); + int hour = parse2(value, 11); + int minute = parse2(value, 14); + int second = 0; + int nano = 0; + int index = 16; + if (index < length && value.charAt(index) == ':') { + second = parse2(value, index + 1); + index += 3; + if (index < length && value.charAt(index) == '.') { + int fractionStart = index + 1; + int fractionEnd = fractionStart; + while (fractionEnd < length && isDigit(value.charAt(fractionEnd))) { + fractionEnd++; + } + if (fractionEnd == fractionStart || fractionEnd - fractionStart > 9) { + throw new IllegalArgumentException(); + } + nano = parseNano(value, fractionStart, fractionEnd); + index = fractionEnd; + } + } + int offsetSeconds = parseOffsetSeconds(value, index); + return OffsetDateTime.of( + year, month, day, hour, minute, second, nano, ZoneOffset.ofTotalSeconds(offsetSeconds)); + } + + private static int parseOffsetSeconds(String value, int index) { + int length = value.length(); + char offset = value.charAt(index); + if (offset == 'Z') { + if (index + 1 != length) { + throw new IllegalArgumentException(); + } + return 0; + } + if (offset != '+' && offset != '-') { + throw new IllegalArgumentException(); + } + if (index + 6 > length || value.charAt(index + 3) != ':') { + throw new IllegalArgumentException(); + } + int hour = parse2(value, index + 1); + int minute = parse2(value, index + 4); + int second = 0; + int end = index + 6; + if (end < length) { + if (end + 3 != length || value.charAt(end) != ':') { + throw new IllegalArgumentException(); + } + second = parse2(value, end + 1); + } + int total = hour * 3600 + minute * 60 + second; + return offset == '-' ? -total : total; + } + + private static int parseNano(String value, int start, int end) { + int nano = 0; + for (int i = start; i < end; i++) { + nano = nano * 10 + value.charAt(i) - '0'; + } + for (int i = end - start; i < 9; i++) { + nano *= 10; + } + return nano; + } + + private static int parse4(String value, int index) { + return parse2(value, index) * 100 + parse2(value, index + 2); + } + + private static int parse2(String value, int index) { + int high = value.charAt(index) - '0'; + int low = value.charAt(index + 1) - '0'; + if (high < 0 || high > 9 || low < 0 || low > 9) { + throw new IllegalArgumentException(); + } + return high * 10 + low; + } + + private static boolean isDigit(char c) { + return c >= '0' && c <= '9'; + } + public static final class AtomicBooleanCodec extends AbstractJsonCodec { public static final AtomicBooleanCodec INSTANCE = new AtomicBooleanCodec(); @@ -1178,6 +1445,160 @@ void writeUtf8NonNull(Utf8JsonWriter writer, Object value, JsonTypeResolver reso } } + public static final class AtomicIntegerArrayCodec extends AbstractJsonCodec { + public static final AtomicIntegerArrayCodec INSTANCE = new AtomicIntegerArrayCodec(); + + @Override + void writeNonNull(JsonWriter writer, Object value, JsonTypeResolver resolver) { + AtomicIntegerArray array = (AtomicIntegerArray) value; + writer.writeArrayStart(); + for (int i = 0, length = array.length(); i < length; i++) { + writer.writeComma(i); + writer.writeInt(array.get(i)); + } + writer.writeArrayEnd(); + } + + @Override + void writeUtf8NonNull(Utf8JsonWriter writer, Object value, JsonTypeResolver resolver) { + AtomicIntegerArray array = (AtomicIntegerArray) value; + writer.writeArrayStart(); + for (int i = 0, length = array.length(); i < length; i++) { + writer.writeComma(i); + writer.writeInt(array.get(i)); + } + writer.writeArrayEnd(); + } + + @Override + Object readNonNull(JsonReader reader, JsonTypeInfo typeInfo, JsonTypeResolver resolver) { + reader.enterDepth(); + reader.expect('['); + if (reader.consume(']')) { + reader.exitDepth(); + return new AtomicIntegerArray(0); + } + int[] values = new int[8]; + int size = 0; + do { + if (reader.tryReadNull()) { + throw new ForyJsonException("Cannot read null into AtomicIntegerArray element"); + } + if (size == values.length) { + values = Arrays.copyOf(values, values.length << 1); + } + values[size++] = reader.readInt(); + } while (reader.consume(',')); + reader.expect(']'); + reader.exitDepth(); + return new AtomicIntegerArray(Arrays.copyOf(values, size)); + } + } + + public static final class AtomicLongArrayCodec extends AbstractJsonCodec { + public static final AtomicLongArrayCodec INSTANCE = new AtomicLongArrayCodec(); + + @Override + void writeNonNull(JsonWriter writer, Object value, JsonTypeResolver resolver) { + AtomicLongArray array = (AtomicLongArray) value; + writer.writeArrayStart(); + for (int i = 0, length = array.length(); i < length; i++) { + writer.writeComma(i); + writer.writeLong(array.get(i)); + } + writer.writeArrayEnd(); + } + + @Override + void writeUtf8NonNull(Utf8JsonWriter writer, Object value, JsonTypeResolver resolver) { + AtomicLongArray array = (AtomicLongArray) value; + writer.writeArrayStart(); + for (int i = 0, length = array.length(); i < length; i++) { + writer.writeComma(i); + writer.writeLong(array.get(i)); + } + writer.writeArrayEnd(); + } + + @Override + Object readNonNull(JsonReader reader, JsonTypeInfo typeInfo, JsonTypeResolver resolver) { + reader.enterDepth(); + reader.expect('['); + if (reader.consume(']')) { + reader.exitDepth(); + return new AtomicLongArray(0); + } + long[] values = new long[8]; + int size = 0; + do { + if (reader.tryReadNull()) { + throw new ForyJsonException("Cannot read null into AtomicLongArray element"); + } + if (size == values.length) { + values = Arrays.copyOf(values, values.length << 1); + } + values[size++] = reader.readLong(); + } while (reader.consume(',')); + reader.expect(']'); + reader.exitDepth(); + return new AtomicLongArray(Arrays.copyOf(values, size)); + } + } + + public static final class AtomicReferenceArrayCodec extends AbstractJsonCodec { + private final JsonTypeInfo valueTypeInfo; + private final JsonCodec valueCodec; + + public AtomicReferenceArrayCodec(java.lang.reflect.Type valueType, JsonTypeResolver resolver) { + Class valueRawType = CodecUtils.rawType(valueType, Object.class); + valueTypeInfo = resolver.getTypeInfo(valueType, valueRawType); + valueCodec = valueTypeInfo.codec(); + } + + @Override + void writeNonNull(JsonWriter writer, Object value, JsonTypeResolver resolver) { + AtomicReferenceArray array = (AtomicReferenceArray) value; + writer.writeArrayStart(); + for (int i = 0, length = array.length(); i < length; i++) { + writer.writeComma(i); + valueCodec.write(writer, array.get(i), resolver); + } + writer.writeArrayEnd(); + } + + @Override + void writeUtf8NonNull(Utf8JsonWriter writer, Object value, JsonTypeResolver resolver) { + AtomicReferenceArray array = (AtomicReferenceArray) value; + writer.writeArrayStart(); + for (int i = 0, length = array.length(); i < length; i++) { + writer.writeComma(i); + valueCodec.writeUtf8(writer, array.get(i), resolver); + } + writer.writeArrayEnd(); + } + + @Override + Object readNonNull(JsonReader reader, JsonTypeInfo typeInfo, JsonTypeResolver resolver) { + reader.enterDepth(); + reader.expect('['); + if (reader.consume(']')) { + reader.exitDepth(); + return new AtomicReferenceArray<>(0); + } + Object[] values = new Object[8]; + int size = 0; + do { + if (size == values.length) { + values = Arrays.copyOf(values, values.length << 1); + } + values[size++] = valueCodec.read(reader, valueTypeInfo, resolver); + } while (reader.consume(',')); + reader.expect(']'); + reader.exitDepth(); + return new AtomicReferenceArray<>(Arrays.copyOf(values, size)); + } + } + public static final class OptionalCodec extends AbstractJsonCodec { private final JsonTypeInfo valueTypeInfo; private final JsonCodec valueCodec; diff --git a/java/fory-json/src/main/java/org/apache/fory/json/codegen/JsonCodegen.java b/java/fory-json/src/main/java/org/apache/fory/json/codegen/JsonCodegen.java index df786e028e..5480f5cdf8 100644 --- a/java/fory-json/src/main/java/org/apache/fory/json/codegen/JsonCodegen.java +++ b/java/fory-json/src/main/java/org/apache/fory/json/codegen/JsonCodegen.java @@ -19,12 +19,15 @@ package org.apache.fory.json.codegen; +import java.lang.invoke.MethodHandle; +import java.lang.invoke.MethodType; import java.lang.reflect.Constructor; import java.lang.reflect.Field; +import java.lang.reflect.Method; import java.lang.reflect.Modifier; import java.util.Collection; import java.util.Map; -import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.ConcurrentHashMap; import org.apache.fory.codegen.CodeGenerator; import org.apache.fory.codegen.CompileUnit; import org.apache.fory.json.ForyJsonException; @@ -40,22 +43,26 @@ import org.apache.fory.json.resolver.JsonTypeResolver; import org.apache.fory.json.writer.StringObjectWriter; import org.apache.fory.json.writer.Utf8ObjectWriter; +import org.apache.fory.platform.AndroidSupport; +import org.apache.fory.platform.internal._JDKAccess; import org.apache.fory.util.record.RecordUtils; public final class JsonCodegen { - static final String PACKAGE = "org.apache.fory.json.codegen"; static final int GENERIC_READER = 0; static final int LATIN1_READER = 1; static final int UTF16_READER = 2; static final int UTF8_READER = 3; - private static final AtomicInteger ID = new AtomicInteger(); + + private static final Map> ID_GENERATOR = new ConcurrentHashMap<>(); final boolean writeNullFields; + private final int configHash; private final CodeGenerator codeGenerator; private final ClassLoader jsonLoader; - public JsonCodegen(boolean writeNullFields) { + public JsonCodegen(boolean writeNullFields, int configHash) { this.writeNullFields = writeNullFields; + this.configHash = configHash; jsonLoader = JsonCodegen.class.getClassLoader(); codeGenerator = new CodeGenerator(jsonLoader); } @@ -78,17 +85,29 @@ boolean record = objectCodec.isRecord(); return null; } } - String className = className(type); + String generatedPackage = CodeGenerator.getPackage(type); JsonCodec[] writeCodecs = writeCodecs(writeProperties); Utf8ObjectWriter utf8Writer = (Utf8ObjectWriter) - compileWriter(className + "_Utf8", type, writeProperties, writeCodecs, true); + compileWriter( + generatedPackage, + className(type, "Utf8"), + type, + writeProperties, + writeCodecs, + true); if (utf8Writer == null) { return null; } StringObjectWriter stringWriter = (StringObjectWriter) - compileWriter(className + "_String", type, writeProperties, writeCodecs, false); + compileWriter( + generatedPackage, + className(type, "String"), + type, + writeProperties, + writeCodecs, + false); if (stringWriter == null) { return null; } @@ -97,7 +116,13 @@ boolean record = objectCodec.isRecord(); ObjectReader reader = (ObjectReader) compileReader( - className + "_Reader", type, readProperties, readCodecs, readObjectCodecs, record); + generatedPackage, + className(type, "Reader"), + type, + readProperties, + readCodecs, + readObjectCodecs, + record); if (reader == null) { return null; } @@ -111,26 +136,37 @@ boolean record = objectCodec.isRecord(); } private Object compileWriter( + String generatedPackage, String className, Class type, JsonFieldInfo[] properties, JsonCodec[] nestedCodecs, boolean utf8) { String code = - new JsonGeneratedCodecBuilder(this, className, type, properties, utf8, true, false) + new JsonGeneratedCodecBuilder( + this, generatedPackage, className, type, properties, utf8, true, false) .genCode(); try { - Class writerClass = compileClass(className, code); - Constructor constructor = - writerClass.getDeclaredConstructor(JsonFieldInfo[].class, JsonCodec[].class); - constructor.setAccessible(true); - return constructor.newInstance(properties, nestedCodecs); + Class writerClass = compileClass(generatedPackage, className, code); + if (AndroidSupport.IS_ANDROID) { + Constructor constructor = + writerClass.getDeclaredConstructor(JsonFieldInfo[].class, JsonCodec[].class); + constructor.setAccessible(true); + return constructor.newInstance(properties, nestedCodecs); + } + MethodHandle constructor = + _JDKAccess._trustedLookup(writerClass) + .findConstructor( + writerClass, + MethodType.methodType(void.class, JsonFieldInfo[].class, JsonCodec[].class)); + return constructor.invoke(properties, nestedCodecs); } catch (Throwable e) { throw new ForyJsonException("Cannot compile generated JSON writer " + className, e); } } private Object compileReader( + String generatedPackage, String className, Class type, JsonFieldInfo[] properties, @@ -138,24 +174,38 @@ private Object compileReader( BaseObjectCodec[] nestedCodecs, boolean record) { String code = - new JsonGeneratedCodecBuilder(this, className, type, properties, false, false, record) + new JsonGeneratedCodecBuilder( + this, generatedPackage, className, type, properties, false, false, record) .genCode(); try { - Class readerClass = compileClass(className, code); - Constructor constructor = - readerClass.getDeclaredConstructor( - JsonFieldInfo[].class, JsonCodec[].class, BaseObjectCodec[].class); - constructor.setAccessible(true); - return constructor.newInstance(properties, readCodecs, nestedCodecs); + Class readerClass = compileClass(generatedPackage, className, code); + if (AndroidSupport.IS_ANDROID) { + Constructor constructor = + readerClass.getDeclaredConstructor( + JsonFieldInfo[].class, JsonCodec[].class, BaseObjectCodec[].class); + constructor.setAccessible(true); + return constructor.newInstance(properties, readCodecs, nestedCodecs); + } + MethodHandle constructor = + _JDKAccess._trustedLookup(readerClass) + .findConstructor( + readerClass, + MethodType.methodType( + void.class, + JsonFieldInfo[].class, + JsonCodec[].class, + BaseObjectCodec[].class)); + return constructor.invoke(properties, readCodecs, nestedCodecs); } catch (Throwable e) { throw new ForyJsonException("Cannot compile generated JSON reader " + className, e); } } - private Class compileClass(String className, String code) throws ClassNotFoundException { - CompileUnit unit = new CompileUnit(PACKAGE, className, code); + private Class compileClass(String generatedPackage, String className, String code) + throws ClassNotFoundException { + CompileUnit unit = new CompileUnit(generatedPackage, className, code); ClassLoader classLoader = codeGenerator.compile(unit); - return classLoader.loadClass(PACKAGE + "." + className); + return classLoader.loadClass(qualifiedClassName(generatedPackage, className)); } private static JsonCodec[] writeCodecs(JsonFieldInfo[] properties) { @@ -203,10 +253,10 @@ static Class readNestedType(JsonFieldInfo property) { private boolean canCompileWrite(JsonFieldInfo property) { Field field = property.writeField(); - if (field == null) { + if (field == null && property.writeGetter() == null) { return false; } - if (!isRecordField(property) && property.writeField() == null) { + if (property.writeGetter() != null && !canCall(property.writeGetter())) { return false; } Class rawType = property.writeRawType(); @@ -221,7 +271,12 @@ private boolean canCompileRead(JsonFieldInfo property, boolean record) { if (!record && property.readAccessor() == null) { return false; } - if (!record && property.readAccessor().coreAccessor() == null) { + if (!record && property.readSetter() != null && !canCall(property.readSetter())) { + return false; + } + if (!record + && property.readSetter() == null + && property.readAccessor().coreAccessor() == null) { return false; } Class rawType = property.readRawType(); @@ -236,6 +291,11 @@ private boolean canCompile(Class type) { return CodeGenerator.sourcePublicAccessible(type) && isVisible(type); } + private boolean canCall(Method method) { + return Modifier.isPublic(method.getModifiers()) + && CodeGenerator.sourcePublicAccessible(method.getDeclaringClass()); + } + private boolean isVisible(Class type) { if (type.isPrimitive()) { return true; @@ -294,6 +354,8 @@ static boolean usesReadCodec(JsonFieldInfo property) { case COLLECTION: case MAP: return true; + case OBJECT: + return !usesReadObjectCodec(property); default: return false; } @@ -306,7 +368,7 @@ static boolean usesReadTypeField(JsonFieldInfo property) { case MAP: return true; case OBJECT: - return usesReadObjectCodec(property); + return true; default: return false; } @@ -323,13 +385,39 @@ static boolean storesReadObjectCodec(Class type, JsonFieldInfo property) { return nestedType != null && nestedType != type; } - private static String className(Class type) { - String name = type.getName().replace('.', '_').replace('$', '_'); - String uniqueId = CodeGenerator.getClassUniqueId(type); - if (uniqueId.isEmpty()) { - uniqueId = String.valueOf(ID.incrementAndGet()); + private String className(Class type, String role) { + String name = simpleClassName(type) + role + "ForyJsonCodec"; + Map subGenerator = + ID_GENERATOR.computeIfAbsent(name, key -> new ConcurrentHashMap<>()); + String key = configHash + "_" + CodeGenerator.getClassUniqueId(type); + Integer id = subGenerator.get(key); + if (id == null) { + synchronized (subGenerator) { + id = subGenerator.computeIfAbsent(key, ignored -> subGenerator.size()); + } } - return "JsonWriter_" + name + "_" + uniqueId; + return id == 0 ? name : name + id; + } + + private static String simpleClassName(Class type) { + String name = type.getName(); + Package declaringPackage = type.getPackage(); + if (declaringPackage != null) { + String prefix = declaringPackage.getName() + "."; + if (name.startsWith(prefix)) { + name = name.substring(prefix.length()); + } + } else { + int separator = name.lastIndexOf('.'); + if (separator >= 0) { + name = name.substring(separator + 1); + } + } + return name.replace('.', '_').replace('$', '_'); + } + + private static String qualifiedClassName(String generatedPackage, String className) { + return generatedPackage.isEmpty() ? className : generatedPackage + "." + className; } private static boolean isPojo(Class type) { diff --git a/java/fory-json/src/main/java/org/apache/fory/json/codegen/JsonGeneratedCodecBuilder.java b/java/fory-json/src/main/java/org/apache/fory/json/codegen/JsonGeneratedCodecBuilder.java index e5e0877993..88a66cc2c4 100644 --- a/java/fory-json/src/main/java/org/apache/fory/json/codegen/JsonGeneratedCodecBuilder.java +++ b/java/fory-json/src/main/java/org/apache/fory/json/codegen/JsonGeneratedCodecBuilder.java @@ -42,6 +42,7 @@ final class JsonGeneratedCodecBuilder extends CodecBuilder { JsonGeneratedCodecBuilder( JsonCodegen codegen, + String generatedPackage, String generatedClassName, Class type, JsonFieldInfo[] properties, @@ -55,7 +56,7 @@ final class JsonGeneratedCodecBuilder extends CodecBuilder { this.utf8 = utf8; this.writer = writer; this.record = record; - ctx.setPackage(JsonCodegen.PACKAGE); + ctx.setPackage(generatedPackage); ctx.setClassName(generatedClassName); ctx.setClassModifiers("final"); ctx.addImports(JsonFieldInfo.class, JsonCodec.class, JsonTypeResolver.class); @@ -81,10 +82,8 @@ public String codecClassName(Class cls) { @Override public String genCode() { return writer - ? new JsonWriterCodegen(codegen) - .genWriterCode(this, generatedClassName, beanClass, properties, utf8) - : new JsonReaderCodegen(codegen) - .genReaderCode(this, generatedClassName, beanClass, properties, record); + ? new JsonWriterCodegen(codegen).genWriterCode(this, beanClass, properties, utf8) + : new JsonReaderCodegen(codegen).genReaderCode(this, beanClass, properties, record); } @Override @@ -119,6 +118,18 @@ private Method recordReadMethod(Field field) { } Expression fieldValue(JsonFieldInfo property, Expression object) { + Method getter = property.writeGetter(); + if (getter != null) { + // JSON writers check the returned member value directly. Requesting expression-level null + // state here only emits an unused boolean for each nullable getter and bloats generated + // object writers enough to hurt C2 inlining. + return new Expression.Invoke( + object, + getter.getName(), + property.name(), + TypeRef.of(getter.getGenericReturnType()), + false); + } return getFieldValue(object, writeDescriptor(property)); } @@ -127,6 +138,15 @@ Expression newObject() { } Expression setField(JsonFieldInfo property, Expression object, Expression value) { + Method setter = property.readSetter(); + if (setter != null) { + Class rawType = setter.getParameterTypes()[0]; + TypeRef typeRef = TypeRef.of(setter.getGenericParameterTypes()[0]); + if (!rawType.isAssignableFrom(value.type().getRawType())) { + value = tryInlineCast(value, typeRef); + } + return new Expression.Invoke(object, setter.getName(), value); + } return setFieldValue( object, readDescriptor(property), diff --git a/java/fory-json/src/main/java/org/apache/fory/json/codegen/JsonReaderCodegen.java b/java/fory-json/src/main/java/org/apache/fory/json/codegen/JsonReaderCodegen.java index 1736168059..4cc03fe04f 100644 --- a/java/fory-json/src/main/java/org/apache/fory/json/codegen/JsonReaderCodegen.java +++ b/java/fory-json/src/main/java/org/apache/fory/json/codegen/JsonReaderCodegen.java @@ -19,6 +19,9 @@ package org.apache.fory.json.codegen; +import static org.apache.fory.codegen.ExpressionUtils.add; +import static org.apache.fory.codegen.ExpressionUtils.inline; + import org.apache.fory.codegen.Code; import org.apache.fory.codegen.CodegenContext; import org.apache.fory.codegen.Expression; @@ -29,6 +32,7 @@ import org.apache.fory.json.meta.JsonAsciiToken; import org.apache.fory.json.meta.JsonFieldInfo; import org.apache.fory.json.meta.JsonFieldKind; +import org.apache.fory.json.meta.JsonFieldTable; import org.apache.fory.json.reader.JsonReader; import org.apache.fory.json.reader.Latin1JsonReader; import org.apache.fory.json.reader.Latin1ObjectReader; @@ -39,6 +43,7 @@ import org.apache.fory.json.reader.Utf8ObjectReader; import org.apache.fory.json.resolver.JsonTypeInfo; import org.apache.fory.json.resolver.JsonTypeResolver; +import org.apache.fory.memory.NativeByteOrder; import org.apache.fory.reflect.TypeRef; final class JsonReaderCodegen { @@ -46,6 +51,12 @@ final class JsonReaderCodegen { private static final int LATIN1_READER = JsonCodegen.LATIN1_READER; private static final int UTF16_READER = JsonCodegen.UTF16_READER; private static final int UTF8_READER = JsonCodegen.UTF8_READER; + private static final int MIN_SPLIT_READ_FIELDS = 8; + private static final int READ_FIELD_GROUP_SIZE = 2; + private static final int READ_FIELD_SWITCH_SIZE = 8; + private static final boolean LITTLE_ENDIAN = NativeByteOrder.IS_LITTLE_ENDIAN; + private static final long UTF16_PAIR_MASK = 0x0000FFFF0000FFFFL; + private static final long UTF16_BYTE_MASK = 0x00FF00FF00FF00FFL; private final JsonCodegen codegen; @@ -63,7 +74,6 @@ private static Class readNestedType(JsonFieldInfo property) { String genReaderCode( JsonGeneratedCodecBuilder builder, - String className, Class type, JsonFieldInfo[] properties, boolean record) { @@ -73,7 +83,8 @@ String genReaderCode( JsonReader.class, Latin1JsonReader.class, Utf16JsonReader.class, - Utf8JsonReader.class); + Utf8JsonReader.class, + JsonFieldTable.class); ctx.implementsInterfaces( ctx.type(ObjectReader.class), ctx.type(Latin1ObjectReader.class), @@ -81,7 +92,9 @@ String genReaderCode( ctx.type(Utf8ObjectReader.class)); ctx.addField(long[].class, "fieldHashes"); for (int i = 0; i < properties.length; i++) { - ctx.addField(JsonFieldInfo.class, "p" + i); + if (usesReadInfo(properties[i])) { + ctx.addField(JsonFieldInfo.class, "p" + i); + } if (usesReadCodec(properties[i])) { ctx.addField(codecFieldType(properties[i].readTypeInfo().codec()), "r" + i); } @@ -101,14 +114,6 @@ String genReaderCode( "codecs", BaseObjectCodec[].class, "objectCodecs"); - addGeneratedMethod( - ctx, - "final", - "fieldIndex", - fieldIndexExpression(properties), - int.class, - long.class, - "fieldHash"); addGeneratedMethod( ctx, "public", @@ -121,11 +126,14 @@ String genReaderCode( "owner", JsonTypeResolver.class, "typeResolver"); + addReadFieldMethods( + ctx, builder, "read", JsonReader.class, type, properties, GENERIC_READER, record); addGeneratedMethod( ctx, "public", "readLatin1", - fastReadExpression(builder, "readLatin1Slow", type, properties, LATIN1_READER, record), + fastReadExpression( + builder, "readLatin1", "readLatin1Slow", type, properties, LATIN1_READER, record), Object.class, Latin1JsonReader.class, "reader", @@ -133,6 +141,25 @@ String genReaderCode( "owner", JsonTypeResolver.class, "typeResolver"); + addFastReadGroupMethods( + ctx, + builder, + "readLatin1", + "readLatin1Slow", + Latin1JsonReader.class, + type, + properties, + LATIN1_READER, + record); + addReadFieldMethods( + ctx, + builder, + "readLatin1", + Latin1JsonReader.class, + type, + properties, + LATIN1_READER, + record); addSlowReadMethods( ctx, builder, @@ -146,7 +173,8 @@ String genReaderCode( ctx, "public", "readUtf16", - fastReadExpression(builder, "readUtf16Slow", type, properties, UTF16_READER, record), + fastReadExpression( + builder, "readUtf16", "readUtf16Slow", type, properties, UTF16_READER, record), Object.class, Utf16JsonReader.class, "reader", @@ -154,6 +182,18 @@ String genReaderCode( "owner", JsonTypeResolver.class, "typeResolver"); + addFastReadGroupMethods( + ctx, + builder, + "readUtf16", + "readUtf16Slow", + Utf16JsonReader.class, + type, + properties, + UTF16_READER, + record); + addReadFieldMethods( + ctx, builder, "readUtf16", Utf16JsonReader.class, type, properties, UTF16_READER, record); addSlowReadMethods( ctx, builder, @@ -167,7 +207,8 @@ String genReaderCode( ctx, "public", "readUtf8", - fastReadExpression(builder, "readUtf8Slow", type, properties, UTF8_READER, record), + fastReadExpression( + builder, "readUtf8", "readUtf8Slow", type, properties, UTF8_READER, record), Object.class, Utf8JsonReader.class, "reader", @@ -175,11 +216,103 @@ String genReaderCode( "owner", JsonTypeResolver.class, "typeResolver"); + addFastReadGroupMethods( + ctx, + builder, + "readUtf8", + "readUtf8Slow", + Utf8JsonReader.class, + type, + properties, + UTF8_READER, + record); + addReadFieldMethods( + ctx, builder, "readUtf8", Utf8JsonReader.class, type, properties, UTF8_READER, record); addSlowReadMethods( ctx, builder, "readUtf8Slow", Utf8JsonReader.class, type, properties, UTF8_READER, record); return ctx.genCode(); } + private void addFastReadGroupMethods( + CodegenContext ctx, + JsonGeneratedCodecBuilder builder, + String readMethod, + String slowMethod, + Class readerType, + Class type, + JsonFieldInfo[] properties, + int readerMode, + boolean record) { + if (!shouldSplitFastRead(properties)) { + return; + } + Class objectType = record ? Object[].class : type; + for (int start = 0; start < properties.length; ) { + int end = readGroupEnd(properties, start); + addGeneratedMethod( + ctx, + "final", + readGroupMethod(readMethod, start), + fastReadGroupExpression( + builder, slowMethod, type, properties, start, end, readerMode, record), + boolean.class, + readerType, + "reader", + BaseObjectCodec.class, + "owner", + JsonTypeResolver.class, + "typeResolver", + objectType, + "object", + long[].class, + "fieldHashes"); + start = end; + } + } + + private void addReadFieldMethods( + CodegenContext ctx, + JsonGeneratedCodecBuilder builder, + String readMethod, + Class readerType, + Class type, + JsonFieldInfo[] properties, + int readerMode, + boolean record) { + if (!shouldSplitFieldSwitch(properties)) { + return; + } + Class objectType = record ? Object[].class : type; + for (int start = 0; start < properties.length; start += READ_FIELD_SWITCH_SIZE) { + int end = Math.min(start + READ_FIELD_SWITCH_SIZE, properties.length); + addGeneratedMethod( + ctx, + "final", + readFieldMethod(readMethod, start), + fieldSwitchRange( + builder, + type, + properties, + start, + end, + readerMode, + objectParam(type, record), + new Reference("fieldIndex", TypeRef.of(int.class)), + record), + void.class, + readerType, + "reader", + BaseObjectCodec.class, + "owner", + JsonTypeResolver.class, + "typeResolver", + objectType, + "object", + int.class, + "fieldIndex"); + } + } + private void addSlowReadMethods( CodegenContext ctx, JsonGeneratedCodecBuilder builder, @@ -259,18 +392,23 @@ private Expression readerConstructorExpression(Class type, JsonFieldInfo[] pr new Expression.Assign( hashes, new Expression.NewArray( - long.class, - new Expression.FieldValue( - propertiesRef, "length", TypeRef.of(int.class), false, true)))); + long.class, + new Expression.FieldValue( + propertiesRef, "length", TypeRef.of(int.class), false, true)) + .inline())); for (int i = 0; i < properties.length; i++) { Expression id = Expression.Literal.ofInt(i); Expression property = new Expression.ArrayValue(propertiesRef, id); - expressions.add( - new Expression.Assign( - new Reference("this.p" + i, TypeRef.of(JsonFieldInfo.class)), property)); + if (usesReadInfo(properties[i])) { + expressions.add( + new Expression.Assign( + new Reference("this.p" + i, TypeRef.of(JsonFieldInfo.class)), property)); + } expressions.add( new Expression.AssignArrayElem( - hashes, new Expression.Invoke(property, "nameHash", TypeRef.of(long.class)), id)); + hashes, + new Expression.Invoke(property, "nameHash", TypeRef.of(long.class)).inline(), + id)); if (usesReadCodec(properties[i])) { Class codecType = codecFieldType(properties[i].readTypeInfo().codec()); expressions.add( @@ -283,7 +421,8 @@ private Expression readerConstructorExpression(Class type, JsonFieldInfo[] pr expressions.add( new Expression.Assign( new Reference("this.t" + i, TypeRef.of(JsonTypeInfo.class)), - new Expression.Invoke(property, "readTypeInfo", TypeRef.of(JsonTypeInfo.class)))); + new Expression.Invoke(property, "readTypeInfo", TypeRef.of(JsonTypeInfo.class)) + .inline())); } if (storesReadObjectCodec(type, properties[i])) { expressions.add( @@ -295,19 +434,6 @@ private Expression readerConstructorExpression(Class type, JsonFieldInfo[] pr return expressions; } - private Expression fieldIndexExpression(JsonFieldInfo[] properties) { - Expression.ListExpression expressions = new Expression.ListExpression(); - Reference fieldHash = new Reference("fieldHash", TypeRef.of(long.class)); - for (int i = 0; i < properties.length; i++) { - expressions.add( - new Expression.If( - eq(fieldHash, Expression.Literal.ofLong(properties[i].nameHash())), - new Expression.Return(Expression.Literal.ofInt(i)))); - } - expressions.add(new Expression.Return(Expression.Literal.ofInt(-1))); - return expressions; - } - private Expression readExpression( JsonGeneratedCodecBuilder builder, Class type, @@ -338,11 +464,16 @@ private Expression readExpression( private Expression fastReadExpression( JsonGeneratedCodecBuilder builder, + String readMethod, String slowMethod, Class type, JsonFieldInfo[] properties, int readerMode, boolean record) { + if (shouldSplitFastRead(properties)) { + return splitFastReadExpression( + builder, readMethod, slowMethod, type, properties, readerMode, record); + } Expression object = objectExpression(builder, record); Expression.ListExpression expressions = new Expression.ListExpression(); expressions.add(object); @@ -371,6 +502,87 @@ private Expression fastReadExpression( return expressions; } + private Expression splitFastReadExpression( + JsonGeneratedCodecBuilder builder, + String readMethod, + String slowMethod, + Class type, + JsonFieldInfo[] properties, + int readerMode, + boolean record) { + Expression object = objectExpression(builder, record); + Expression.ListExpression expressions = new Expression.ListExpression(); + expressions.add(object); + expressions.add(expectExpr(readerMode, '{')); + expressions.add(new Expression.If(consumeExpr(readerMode, '}'), returnObject(object, record))); + Expression hashes = + new Expression.Variable("localFieldHashes", fieldRef("fieldHashes", long[].class)); + expressions.add(hashes); + for (int start = 0; start < properties.length; ) { + int end = readGroupEnd(properties, start); + Expression groupCall = + inline( + new Expression.Invoke( + new Reference("this", TypeRef.of(Object.class)), + readGroupMethod(readMethod, start), + "", + TypeRef.of(boolean.class), + false, + false, + readerRef(readerMode), + ownerRef(), + typeResolverRef(), + object, + hashes)); + expressions.add(new Expression.If(not(groupCall), returnObject(object, record))); + start = end; + } + expressions.add(returnObject(object, record)); + return expressions; + } + + private Expression fastReadGroupExpression( + JsonGeneratedCodecBuilder builder, + String slowMethod, + Class type, + JsonFieldInfo[] properties, + int start, + int end, + int readerMode, + boolean record) { + Expression object = objectParam(type, record); + Expression hashes = new Reference("fieldHashes", TypeRef.of(long[].class)); + Expression[] skips = new Expression[properties.length]; + Expression.ListExpression expressions = new Expression.ListExpression(); + for (int i = start + 1; i < end; i++) { + skips[i] = new Expression.Variable("skip" + i, Expression.Literal.False); + expressions.add(skips[i]); + } + for (int i = start; i < end; i++) { + Expression read = + fastReadField( + builder, + slowMethod, + type, + properties, + i, + end, + true, + readerMode, + object, + hashes, + skips, + record); + expressions.add(i == start ? read : new Expression.If(not(skips[i]), read)); + } + if (end < properties.length) { + expressions.add(returnTrue()); + } else { + expressions.add(returnFalse()); + } + return expressions; + } + private Expression fastReadField( JsonGeneratedCodecBuilder builder, String slowMethod, @@ -382,9 +594,37 @@ private Expression fastReadField( Expression hashes, Expression[] skips, boolean record) { - if (isPackedName(properties[index].name())) { - return new Expression.If( - tryReadNextFieldNameColon(readerMode, properties[index]), + return fastReadField( + builder, + slowMethod, + type, + properties, + index, + properties.length, + false, + readerMode, + object, + hashes, + skips, + record); + } + + private Expression fastReadField( + JsonGeneratedCodecBuilder builder, + String slowMethod, + Class type, + JsonFieldInfo[] properties, + int index, + int groupEnd, + boolean groupHelper, + int readerMode, + Expression object, + Expression hashes, + Expression[] skips, + boolean record) { + if (isDirectName(readerMode, properties[index].name(), usesTokenValueRead(readerMode))) { + return statementIf( + tryReadNextFieldNameColon(readerMode, properties[index], usesTokenValueRead(readerMode)), new Expression.ListExpression( readField( builder, @@ -395,21 +635,43 @@ private Expression fastReadField( object, record, usesTokenValueRead(readerMode)), - fieldEnd(slowMethod, properties.length, index, readerMode, object, record)), + fieldEnd( + slowMethod, + properties.length, + groupEnd, + groupHelper, + index, + readerMode, + object, + record)), nextDirectFallback( builder, slowMethod, type, properties, index, + groupEnd, + groupHelper, readerMode, object, hashes, skips, - record)); + record), + groupHelper); } return nextDirectFallback( - builder, slowMethod, type, properties, index, readerMode, object, hashes, skips, record); + builder, + slowMethod, + type, + properties, + index, + groupEnd, + groupHelper, + readerMode, + object, + hashes, + skips, + record); } private Expression nextDirectFallback( @@ -418,15 +680,19 @@ private Expression nextDirectFallback( Class type, JsonFieldInfo[] properties, int index, + int groupEnd, + boolean groupHelper, int readerMode, Expression object, Expression hashes, Expression[] skips, boolean record) { int nextIndex = index + 1; - if (nextIndex < properties.length && isPackedName(properties[nextIndex].name())) { - return new Expression.If( - tryReadNextFieldNameColon(readerMode, properties[nextIndex]), + if (nextIndex < groupEnd + && isDirectName(readerMode, properties[nextIndex].name(), usesTokenValueRead(readerMode))) { + return statementIf( + tryReadNextFieldNameColon( + readerMode, properties[nextIndex], usesTokenValueRead(readerMode)), new Expression.ListExpression( readField( builder, @@ -437,22 +703,44 @@ private Expression nextDirectFallback( object, record, usesTokenValueRead(readerMode)), - fieldEnd(slowMethod, properties.length, nextIndex, readerMode, object, record), - new Expression.Assign(skips[nextIndex], Expression.Literal.True)), + new Expression.Assign(skips[nextIndex], Expression.Literal.True), + fieldEnd( + slowMethod, + properties.length, + groupEnd, + groupHelper, + nextIndex, + readerMode, + object, + record)), hashFallback( builder, slowMethod, type, properties, index, + groupEnd, + groupHelper, readerMode, object, hashes, skips, - record)); + record), + groupHelper); } return hashFallback( - builder, slowMethod, type, properties, index, readerMode, object, hashes, skips, record); + builder, + slowMethod, + type, + properties, + index, + groupEnd, + groupHelper, + readerMode, + object, + hashes, + skips, + record); } private Expression hashFallback( @@ -461,6 +749,8 @@ private Expression hashFallback( Class type, JsonFieldInfo[] properties, int index, + int groupEnd, + boolean groupHelper, int readerMode, Expression object, Expression hashes, @@ -475,6 +765,8 @@ private Expression hashFallback( type, properties, index, + groupEnd, + groupHelper, readerMode, object, hashes, @@ -489,6 +781,8 @@ private Expression fastReadFieldFromHash( Class type, JsonFieldInfo[] properties, int index, + int groupEnd, + boolean groupHelper, int readerMode, Expression object, Expression hashes, @@ -496,9 +790,9 @@ private Expression fastReadFieldFromHash( Expression fieldHash, boolean record) { Expression fallback; - if (index + 1 < properties.length) { + if (index + 1 < groupEnd) { fallback = - new Expression.If( + statementIf( eq(fieldHash, arrayValue(hashes, index + 1)), new Expression.ListExpression( expectExpr(readerMode, ':'), @@ -511,19 +805,40 @@ private Expression fastReadFieldFromHash( object, record, false), - fieldEnd(slowMethod, properties.length, index + 1, readerMode, object, record), - new Expression.Assign(skips[index + 1], Expression.Literal.True)), - slowConsumedReturn(slowMethod, index, fieldIndexInvoke(fieldHash), object, record)); + new Expression.Assign(skips[index + 1], Expression.Literal.True), + fieldEnd( + slowMethod, + properties.length, + groupEnd, + groupHelper, + index + 1, + readerMode, + object, + record)), + slowConsumedReturn( + slowMethod, index, fieldIndexInvoke(fieldHash), object, record, groupHelper), + groupHelper); } else { - fallback = slowConsumedReturn(slowMethod, index, fieldIndexInvoke(fieldHash), object, record); + fallback = + slowConsumedReturn( + slowMethod, index, fieldIndexInvoke(fieldHash), object, record, groupHelper); } - return new Expression.If( + return statementIf( ne(fieldHash, arrayValue(hashes, index)), fallback, new Expression.ListExpression( expectExpr(readerMode, ':'), readField(builder, type, properties[index], index, readerMode, object, record, false), - fieldEnd(slowMethod, properties.length, index, readerMode, object, record))); + fieldEnd( + slowMethod, + properties.length, + groupEnd, + groupHelper, + index, + readerMode, + object, + record)), + groupHelper); } private static boolean isPackedName(String name) { @@ -531,9 +846,32 @@ private static boolean isPackedName(String name) { if (length == 0 || length > Long.BYTES) { return false; } + return isAsciiName(name); + } + + private static boolean isDirectName(int readerMode, String name, boolean tokenValueRead) { + if (readerMode == LATIN1_READER || readerMode == UTF8_READER) { + return JsonAsciiToken.isLongPackable(fieldNameToken(name)); + } + if (readerMode == UTF16_READER && tokenValueRead) { + return isUtf16FieldNameToken(name); + } + return isPackedName(name); + } + + private static boolean isUtf16FieldNameToken(String name) { + int length = name.length(); + if (length == 0 || length + 3 > 12) { + return false; + } + return isAsciiName(name); + } + + private static boolean isAsciiName(String name) { + int length = name.length(); for (int i = 0; i < length; i++) { char ch = name.charAt(i); - if (ch == 0 || ch > 0xFF) { + if (ch == '"' || ch == '\\' || ch < 0x20 || ch > 0xFF) { return false; } } @@ -544,6 +882,72 @@ private static long packedNameMask(int length) { return length == Long.BYTES ? -1L : (1L << (length << 3)) - 1L; } + private static boolean shouldSplitFastRead(JsonFieldInfo[] properties) { + return properties.length >= MIN_SPLIT_READ_FIELDS; + } + + private static String readGroupMethod(String readMethod, int start) { + return readMethod + "Group" + start; + } + + private static String readFieldMethod(String readMethod, int start) { + return readMethod + "Field" + start; + } + + private static int readGroupEnd(JsonFieldInfo[] properties, int start) { + int end = start + 1; + while (end < properties.length + && end - start < READ_FIELD_GROUP_SIZE + && canPairReadFields(properties[end - 1], properties[end])) { + end++; + } + return end; + } + + private static boolean canPairReadFields(JsonFieldInfo left, JsonFieldInfo right) { + JsonFieldKind leftKind = left.readKind(); + JsonFieldKind rightKind = right.readKind(); + if (leftKind == null || rightKind == null) { + return false; + } + // Fast-read fallback branches duplicate some field reads in each group. Keep method-size + // estimation conservative so generated helpers stay close to the C2-friendly target. + if (leftKind == JsonFieldKind.ENUM + || rightKind == JsonFieldKind.ENUM + || leftKind == JsonFieldKind.COLLECTION + || rightKind == JsonFieldKind.COLLECTION + || leftKind == JsonFieldKind.MAP + || rightKind == JsonFieldKind.MAP) { + return false; + } + if (leftKind == JsonFieldKind.ARRAY || rightKind == JsonFieldKind.ARRAY) { + return false; + } + if (leftKind == JsonFieldKind.OBJECT || rightKind == JsonFieldKind.OBJECT) { + return leftKind == JsonFieldKind.BOOLEAN || rightKind == JsonFieldKind.BOOLEAN; + } + return true; + } + + private static String readMethod(int readerMode) { + switch (readerMode) { + case GENERIC_READER: + return "read"; + case LATIN1_READER: + return "readLatin1"; + case UTF16_READER: + return "readUtf16"; + case UTF8_READER: + return "readUtf8"; + default: + throw new IllegalArgumentException(String.valueOf(readerMode)); + } + } + + private static boolean shouldSplitFieldSwitch(JsonFieldInfo[] properties) { + return properties.length > READ_FIELD_SWITCH_SIZE; + } + private Expression objectExpression(JsonGeneratedCodecBuilder builder, boolean record) { if (record) { return new Expression.Variable( @@ -562,6 +966,22 @@ private Expression returnObject(Expression object, boolean record) { return new Expression.Return(object); } + private static Expression returnTrue() { + return new Expression.Return(Expression.Literal.True); + } + + private static Expression returnFalse() { + return new Expression.Return(Expression.Literal.False); + } + + private static Expression statementIf( + Expression predicate, Expression trueExpr, Expression falseExpr, boolean statementOnly) { + if (statementOnly) { + return new Expression.If(predicate, trueExpr, falseExpr, false, TypeRef.of(void.class)); + } + return new Expression.If(predicate, trueExpr, falseExpr); + } + private Expression slowConsumedReturn( String slowMethod, int index, Expression firstFieldIndex, Expression object, boolean record) { return new Expression.ListExpression( @@ -569,6 +989,36 @@ private Expression slowConsumedReturn( returnObject(object, record)); } + private Expression slowConsumedReturn( + String slowMethod, + int index, + Expression firstFieldIndex, + Expression object, + boolean record, + boolean groupHelper) { + if (!groupHelper) { + return slowConsumedReturn(slowMethod, index, firstFieldIndex, object, record); + } + return new Expression.ListExpression( + slowCall(slowMethod, object, Expression.Literal.ofInt(index), firstFieldIndex), + returnFalse()); + } + + private Expression fieldEnd( + String slowMethod, + int propertyCount, + int groupEnd, + boolean groupHelper, + int index, + int readerMode, + Expression object, + boolean record) { + if (!groupHelper) { + return fieldEnd(slowMethod, propertyCount, index, readerMode, object, record); + } + return fastReadGroupEnd(slowMethod, propertyCount, index, readerMode, object); + } + private Expression fieldEnd( String slowMethod, int propertyCount, @@ -585,6 +1035,17 @@ private Expression fieldEnd( slowCall(slowMethod, object, Expression.Literal.ofInt(propertyCount))); } + private Expression fastReadGroupEnd( + String slowMethod, int propertyCount, int index, int readerMode, Expression object) { + if (index + 1 < propertyCount) { + return new Expression.If(not(consumeCommaOrEndObjectExpr(readerMode)), returnFalse()); + } + return new Expression.ListExpression( + new Expression.If( + consumeCommaOrEndObjectExpr(readerMode), + slowCall(slowMethod, object, Expression.Literal.ofInt(propertyCount)))); + } + private Expression slowReadExpression( JsonGeneratedCodecBuilder builder, Class type, @@ -664,9 +1125,51 @@ private Expression fieldSwitch( Expression object, Expression fieldIndex, boolean record) { - Expression.Switch.Case[] cases = new Expression.Switch.Case[properties.length]; - for (int i = 0; i < properties.length; i++) { - cases[i] = + if (shouldSplitFieldSwitch(properties)) { + int chunks = (properties.length + READ_FIELD_SWITCH_SIZE - 1) / READ_FIELD_SWITCH_SIZE; + Expression.Switch.Case[] cases = new Expression.Switch.Case[chunks]; + for (int i = 0, start = 0; i < chunks; i++, start += READ_FIELD_SWITCH_SIZE) { + cases[i] = + new Expression.Switch.Case( + i, + new Expression.ListExpression( + new Expression.Invoke( + new Reference("this", TypeRef.of(Object.class)), + readFieldMethod(readMethod(readerMode), start), + "", + TypeRef.of(void.class), + false, + false, + readerRef(readerMode), + ownerRef(), + typeResolverRef(), + object, + fieldIndex), + new Expression.Break())); + } + Expression chunkIndex = + new Expression.Arithmetic( + true, "/", fieldIndex, Expression.Literal.ofInt(READ_FIELD_SWITCH_SIZE)); + return new Expression.Switch( + chunkIndex, cases, new Expression.Invoke(readerRef(readerMode), "skipValue")); + } + return fieldSwitchRange( + builder, type, properties, 0, properties.length, readerMode, object, fieldIndex, record); + } + + private Expression fieldSwitchRange( + JsonGeneratedCodecBuilder builder, + Class type, + JsonFieldInfo[] properties, + int start, + int end, + int readerMode, + Expression object, + Expression fieldIndex, + boolean record) { + Expression.Switch.Case[] cases = new Expression.Switch.Case[end - start]; + for (int i = start; i < end; i++) { + cases[i - start] = new Expression.Switch.Case( i, new Expression.ListExpression( @@ -680,8 +1183,7 @@ private Expression fieldSwitch( private Expression updateExpectedIndex(Expression expectedIndex, Expression fieldIndex) { return new Expression.If( ge(fieldIndex, Expression.Literal.ofInt(0)), - new Expression.Assign( - expectedIndex, new Expression.Add(fieldIndex, Expression.Literal.ofInt(1)))); + new Expression.Assign(expectedIndex, add(fieldIndex, Expression.Literal.ofInt(1)))); } private Expression fieldIndexValue( @@ -801,6 +1303,14 @@ private static Expression readLongExpr(int readerMode, boolean tokenValueRead) { .inline(); } + private static Expression readDoubleExpr(int readerMode, boolean tokenValueRead) { + return new Expression.Invoke( + readerRef(readerMode), + readDoubleMethod(readerMode, tokenValueRead), + TypeRef.of(double.class)) + .inline(); + } + private static Expression readStringExpr(int readerMode) { return readStringExpr(readerMode, false); } @@ -835,6 +1345,13 @@ private static String readLongMethod(int readerMode, boolean tokenValueRead) { return tokenValueRead ? "readLongTokenValue" : "readNextLongValue"; } + private static String readDoubleMethod(int readerMode, boolean tokenValueRead) { + if (readerMode == LATIN1_READER || readerMode == UTF8_READER) { + return tokenValueRead ? "readDoubleTokenValue" : "readNextDoubleValue"; + } + return "readDouble"; + } + private static String readStringMethod(int readerMode, boolean tokenValueRead) { if (readerMode == GENERIC_READER) { return "readNullableString"; @@ -843,7 +1360,7 @@ private static String readStringMethod(int readerMode, boolean tokenValueRead) { } private static boolean usesTokenValueRead(int readerMode) { - return readerMode == LATIN1_READER || readerMode == UTF8_READER; + return readerMode == LATIN1_READER || readerMode == UTF16_READER || readerMode == UTF8_READER; } private static Expression readFieldNameHash(int readerMode, String namePrefix) { @@ -853,17 +1370,16 @@ private static Expression readFieldNameHash(int readerMode, String namePrefix) { private static Expression fieldIndexInvoke(Expression fieldHash) { return new Expression.Invoke( - new Reference("this", TypeRef.of(Object.class)), - "fieldIndex", - "", + new Expression.Invoke(ownerRef(), "readTable", TypeRef.of(JsonFieldTable.class)), + "index", TypeRef.of(int.class), - false, - false, + true, fieldHash) .inline(); } - private static Expression tryReadNextFieldNameColon(int readerMode, JsonFieldInfo property) { + private static Expression tryReadNextFieldNameColon( + int readerMode, JsonFieldInfo property, boolean tokenValueRead) { if (readerMode == LATIN1_READER || readerMode == UTF8_READER) { String name = property.name(); String token = fieldNameToken(name); @@ -881,6 +1397,17 @@ private static Expression tryReadNextFieldNameColon(int readerMode, JsonFieldInf Expression.Literal.ofInt(tokenLength)) .inline(); } + if (suffixLength > 3) { + return new Expression.Invoke( + readerRef(readerMode), + "tryReadNextFieldNameToken8", + TypeRef.of(boolean.class), + Expression.Literal.ofLong(JsonAsciiToken.prefix(token)), + Expression.Literal.ofLong(JsonAsciiToken.suffixLong(token)), + Expression.Literal.ofLong(JsonAsciiToken.suffixMask(tokenLength)), + Expression.Literal.ofInt(tokenLength)) + .inline(); + } return new Expression.Invoke( readerRef(readerMode), "tryReadNextFieldNameToken" + suffixLength, @@ -891,6 +1418,50 @@ private static Expression tryReadNextFieldNameColon(int readerMode, JsonFieldInf Expression.Literal.ofInt(tokenLength)) .inline(); } + if (readerMode == UTF16_READER) { + String name = property.name(); + int length = name.length(); + if (tokenValueRead) { + String token = fieldNameToken(name); + int tokenLength = token.length(); + int tailLength = Math.max(0, tokenLength - 4); + if (tokenLength <= 8) { + return new Expression.Invoke( + readerRef(readerMode), + "tryReadNextFieldNameUtf16Token2", + TypeRef.of(boolean.class), + Expression.Literal.ofLong(utf16TokenWord(token, 0, Math.min(tokenLength, 4))), + Expression.Literal.ofLong(utf16WordMask(Math.min(tokenLength, 4))), + Expression.Literal.ofLong( + tailLength == 0 ? 0 : utf16TokenWord(token, 4, tailLength)), + Expression.Literal.ofLong(tailLength == 0 ? 0 : utf16WordMask(tailLength)), + Expression.Literal.ofInt(tokenLength)) + .inline(); + } + return new Expression.Invoke( + readerRef(readerMode), + "tryReadNextFieldNameUtf16Token3", + TypeRef.of(boolean.class), + Expression.Literal.ofLong(utf16TokenWord(token, 0, 4)), + Expression.Literal.ofLong(utf16TokenWord(token, 4, 4)), + Expression.Literal.ofLong(utf16TokenWord(token, 8, tokenLength - 8)), + Expression.Literal.ofInt(tokenLength)) + .inline(); + } + int tailLength = Math.max(0, length - 4); + return new Expression.Invoke( + readerRef(readerMode), + "tryReadNextFieldNameUtf16", + TypeRef.of(boolean.class), + Expression.Literal.ofLong(property.nameHash()), + Expression.Literal.ofLong(packedNameMask(length)), + Expression.Literal.ofLong(utf16NameWord(name, 0, Math.min(length, 4))), + Expression.Literal.ofLong(utf16WordMask(Math.min(length, 4))), + Expression.Literal.ofLong(tailLength == 0 ? 0 : utf16NameWord(name, 4, tailLength)), + Expression.Literal.ofLong(tailLength == 0 ? 0 : utf16WordMask(tailLength)), + Expression.Literal.ofInt(length)) + .inline(); + } return new Expression.Invoke( readerRef(readerMode), "tryReadNextFieldNameColon", @@ -901,6 +1472,28 @@ private static Expression tryReadNextFieldNameColon(int readerMode, JsonFieldInf .inline(); } + private static long utf16NameWord(String name, int start, int length) { + return utf16TokenWord(name, start, length); + } + + private static long utf16TokenWord(String token, int start, int length) { + long value = 0; + for (int i = 0; i < length; i++) { + value |= (long) (token.charAt(start + i) & 0xFF) << (i << 3); + } + long word = spreadLatin1ToUtf16(value); + return LITTLE_ENDIAN ? word : word << 8; + } + + private static long utf16WordMask(int length) { + return length == 4 ? -1L : (1L << (length << 4)) - 1; + } + + private static long spreadLatin1ToUtf16(long value) { + value = (value | (value << 16)) & UTF16_PAIR_MASK; + return (value | (value << 8)) & UTF16_BYTE_MASK; + } + private static Expression tryReadNextStringToken(int readerMode, String token) { int tokenLength = token.length(); int suffixLength = JsonAsciiToken.suffixLength(tokenLength); @@ -997,41 +1590,52 @@ private static Expression not(Expression expression) { return new Expression.Not(expression); } - private static boolean usesWriteCodec(JsonFieldInfo property) { - switch (property.writeKind()) { + private static boolean usesReadCodec(JsonFieldInfo property) { + switch (property.readKind()) { + case ENUM: case ARRAY: + case COLLECTION: case MAP: - case OBJECT: return true; - case COLLECTION: - return property.writeElementRawType() != String.class; + case OBJECT: + return !usesReadObjectCodec(property); default: return false; } } - private static boolean usesReadCodec(JsonFieldInfo property) { + private static boolean usesReadTypeField(JsonFieldInfo property) { switch (property.readKind()) { - case ENUM: case ARRAY: case COLLECTION: case MAP: return true; + case OBJECT: + return true; default: return false; } } - private static boolean usesReadTypeField(JsonFieldInfo property) { + private static boolean usesReadInfo(JsonFieldInfo property) { switch (property.readKind()) { - case ARRAY: + case BOOLEAN: + case INT: + case LONG: + case DOUBLE: + case STRING: + case ENUM: case COLLECTION: + case ARRAY: case MAP: - return true; case OBJECT: - return usesReadObjectCodec(property); - default: return false; + case BYTE: + case SHORT: + case FLOAT: + case CHAR: + default: + return true; } } @@ -1066,6 +1670,8 @@ private Expression readField( return readInt(builder, property, rawType, readerMode, object, tokenValueRead); case LONG: return readLong(builder, property, rawType, readerMode, object, tokenValueRead); + case DOUBLE: + return readDouble(builder, property, rawType, readerMode, object, tokenValueRead); case STRING: return builder.setField(property, object, readStringExpr(readerMode, tokenValueRead)); case ENUM: @@ -1102,6 +1708,8 @@ private Expression readRecordField( return readRecordInt(rawType, id, readerMode, object, tokenValueRead); case LONG: return readRecordLong(rawType, id, readerMode, object, tokenValueRead); + case DOUBLE: + return readRecordDouble(rawType, id, readerMode, object, tokenValueRead); case STRING: return assignRecord(object, id, readStringExpr(readerMode, tokenValueRead)); case ENUM: @@ -1163,6 +1771,18 @@ private static Expression readRecordLong( assignRecord(object, id, value)); } + private static Expression readRecordDouble( + Class rawType, int id, int readerMode, Expression object, boolean tokenValueRead) { + Expression value = box(Double.class, readDoubleExpr(readerMode, tokenValueRead)); + if (rawType.isPrimitive()) { + return assignRecord(object, id, value); + } + return new Expression.If( + tryReadNullExpr(readerMode), + assignRecord(object, id, new Expression.Null(TypeRef.of(Double.class), false)), + assignRecord(object, id, value)); + } + private static Expression readRecordEnum( int id, int readerMode, Expression object, boolean tokenValueRead) { return new Expression.If( @@ -1175,16 +1795,7 @@ private static Expression readRecordObject( Class type, JsonFieldInfo property, int id, int readerMode, Expression object) { if (property.readRawType() == Object.class || !(property.readTypeInfo().codec() instanceof BaseObjectCodec)) { - return assignRecord( - object, - id, - new Expression.Invoke( - fieldRef("p" + id, JsonFieldInfo.class), - "readValue", - TypeRef.of(Object.class), - true, - readerRef(readerMode), - typeResolverRef())); + return assignRecord(object, id, readResolvedValue(property, id, readerMode)); } return new Expression.If( tryReadNullExpr(readerMode), @@ -1254,6 +1865,23 @@ private static Expression readLong( property, object, box(Long.class, readLongExpr(readerMode, tokenValueRead)))); } + private static Expression readDouble( + JsonGeneratedCodecBuilder builder, + JsonFieldInfo property, + Class rawType, + int readerMode, + Expression object, + boolean tokenValueRead) { + if (rawType.isPrimitive()) { + return builder.setField(property, object, readDoubleExpr(readerMode, tokenValueRead)); + } + return new Expression.If( + tryReadNullExpr(readerMode), + builder.setNull(property, object), + builder.setField( + property, object, box(Double.class, readDoubleExpr(readerMode, tokenValueRead)))); + } + private static Expression readEnum( JsonGeneratedCodecBuilder builder, JsonFieldInfo property, @@ -1331,12 +1959,7 @@ private static Expression readObject( Expression object) { if (property.readRawType() == Object.class || !(property.readTypeInfo().codec() instanceof BaseObjectCodec)) { - return new Expression.Invoke( - fieldRef("p" + id, JsonFieldInfo.class), - "read", - readerRef(readerMode), - object, - typeResolverRef()); + return readResolvedField(builder, property, id, readerMode, object); } return new Expression.If( tryReadNullExpr(readerMode), @@ -1361,40 +1984,46 @@ private static Expression readEnumValue( private static Expression readEnumValue( Class enumType, int id, int readerMode, boolean tokenValueRead, boolean hashFallback) { return new Expression.Cast( - new Expression.Invoke( - fieldRef("r" + id, JsonCodec.class), - readEnumMethod(readerMode, tokenValueRead, hashFallback), - "", - TypeRef.of(Object.class), - true, - false, - readerRef(readerMode)), + inline( + new Expression.Invoke( + fieldRef("r" + id, JsonCodec.class), + readEnumMethod(readerMode, tokenValueRead, hashFallback), + "", + TypeRef.of(Object.class), + false, + false, + readerRef(readerMode))), TypeRef.of(enumType)); } private static Expression readResolvedValue(JsonFieldInfo property, int id, int readerMode) { + // Generated readers either branch on JSON null before calling a non-null codec path, or assign + // nullable reference results directly. Requesting expression null-state here only emits dead + // boolean locals around codec calls and bloats hot generated reader methods. return new Expression.Cast( - new Expression.Invoke( - fieldRef("r" + id, JsonCodec.class), - readObjectMethod(readerMode), - TypeRef.of(Object.class), - true, - readerRef(readerMode), - fieldRef("t" + id, JsonTypeInfo.class), - typeResolverRef()), + inline( + new Expression.Invoke( + fieldRef("r" + id, JsonCodec.class), + readObjectMethod(readerMode), + TypeRef.of(Object.class), + false, + readerRef(readerMode), + fieldRef("t" + id, JsonTypeInfo.class), + typeResolverRef())), TypeRef.of(property.readRawType())); } private static Expression readCollectionValue(JsonFieldInfo property, int id, int readerMode) { return new Expression.Cast( - new Expression.Invoke( - fieldRef("r" + id, CollectionCodec.class), - readObjectNonNullMethod(readerMode), - TypeRef.of(Object.class), - true, - readerRef(readerMode), - fieldRef("t" + id, JsonTypeInfo.class), - typeResolverRef()), + inline( + new Expression.Invoke( + fieldRef("r" + id, CollectionCodec.class), + readObjectNonNullMethod(readerMode), + TypeRef.of(Object.class), + false, + readerRef(readerMode), + fieldRef("t" + id, JsonTypeInfo.class), + typeResolverRef())), TypeRef.of(property.readRawType())); } @@ -1403,14 +2032,15 @@ private static Expression readObjectValue( Expression codec = property.readRawType() == type ? ownerRef() : fieldRef("c" + id, BaseObjectCodec.class); return new Expression.Cast( - new Expression.Invoke( - codec, - readObjectNonNullMethod(readerMode), - TypeRef.of(Object.class), - true, - readerRef(readerMode), - fieldRef("t" + id, JsonTypeInfo.class), - typeResolverRef()), + inline( + new Expression.Invoke( + codec, + readObjectNonNullMethod(readerMode), + TypeRef.of(Object.class), + false, + readerRef(readerMode), + fieldRef("t" + id, JsonTypeInfo.class), + typeResolverRef())), TypeRef.of(property.readRawType())); } diff --git a/java/fory-json/src/main/java/org/apache/fory/json/codegen/JsonWriterCodegen.java b/java/fory-json/src/main/java/org/apache/fory/json/codegen/JsonWriterCodegen.java index 689dfa2951..6a4decce32 100644 --- a/java/fory-json/src/main/java/org/apache/fory/json/codegen/JsonWriterCodegen.java +++ b/java/fory-json/src/main/java/org/apache/fory/json/codegen/JsonWriterCodegen.java @@ -19,16 +19,27 @@ package org.apache.fory.json.codegen; +import static org.apache.fory.codegen.ExpressionUtils.add; +import static org.apache.fory.codegen.ExpressionUtils.cast; +import static org.apache.fory.codegen.ExpressionUtils.inline; + +import java.math.BigDecimal; +import java.time.LocalDate; +import java.time.OffsetDateTime; +import java.util.ArrayList; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.UUID; import org.apache.fory.codegen.Code; import org.apache.fory.codegen.CodegenContext; import org.apache.fory.codegen.Expression; import org.apache.fory.codegen.Expression.Reference; +import org.apache.fory.codegen.ExpressionOptimizer; import org.apache.fory.json.ForyJsonException; import org.apache.fory.json.codec.JsonCodec; import org.apache.fory.json.meta.JsonFieldInfo; import org.apache.fory.json.meta.JsonFieldKind; import org.apache.fory.json.resolver.JsonTypeResolver; -import org.apache.fory.json.writer.GeneratedObjectWriter; import org.apache.fory.json.writer.StringJsonWriter; import org.apache.fory.json.writer.StringObjectWriter; import org.apache.fory.json.writer.Utf8JsonWriter; @@ -36,6 +47,13 @@ import org.apache.fory.reflect.TypeRef; final class JsonWriterCodegen { + private static final int MIN_STRING_SPLIT_MEMBERS = 10; + private static final int MIN_UTF8_SPLIT_MEMBERS = 12; + // Keep generated member helpers below C2's big-method range without fragmenting them into + // tiny calls. This mirrors Fory core's object-codec split strategy for large generated codecs. + private static final int MEMBER_GROUP_SIZE = 8; + private static final int INLINE_TAIL_MEMBERS = 4; + private final JsonCodegen codegen; private final boolean writeNullFields; @@ -92,41 +110,40 @@ private static void addGeneratedMethod( } String genWriterCode( - JsonGeneratedCodecBuilder builder, - String className, - Class type, - JsonFieldInfo[] properties, - boolean utf8) { + JsonGeneratedCodecBuilder builder, Class type, JsonFieldInfo[] properties, boolean utf8) { CodegenContext ctx = builder.context(); ctx.addImports(StringJsonWriter.class, Utf8JsonWriter.class); - ctx.extendsClasses(ctx.type(GeneratedObjectWriter.class)); ctx.implementsInterfaces(ctx.type(utf8 ? Utf8ObjectWriter.class : StringObjectWriter.class)); boolean objectStartFused = canFuseObjectStart(properties); - boolean[] useInfo = new boolean[properties.length]; - boolean[] usePrefix = new boolean[properties.length]; + StringPrefixFields stringPrefixFields = + utf8 ? null : stringPrefixFields(properties, objectStartFused); for (int i = 0; i < properties.length; i++) { JsonFieldInfo property = properties[i]; - useInfo[i] = true; - usePrefix[i] = usesPrefix(property); - if (useInfo[i]) { + if (usesWriteInfo(property)) { ctx.addField(JsonFieldInfo.class, "p" + i); } if (usesWriteCodec(property)) { ctx.addField(codecFieldType(property.writeTypeInfo().codec()), "c" + i); } - if (usePrefix[i]) { + if (usesPrefix(property)) { if (utf8) { ctx.addField(byte[].class, "u" + i); ctx.addField(byte[].class, "uc" + i); } else { ctx.addField(byte[].class, "s" + i); ctx.addField(byte[].class, "sc" + i); + if (stringPrefixFields.name[i]) { + ctx.addField(byte[].class, "s16" + i); + } + if (stringPrefixFields.comma[i]) { + ctx.addField(byte[].class, "sc16" + i); + } } } } addGeneratedConstructor( ctx, - writerConstructorExpression(properties, utf8), + writerConstructorExpression(properties, utf8, stringPrefixFields), JsonFieldInfo[].class, "properties", JsonCodec[].class, @@ -146,23 +163,74 @@ String genWriterCode( return ctx.genCode(); } + private StringPrefixFields stringPrefixFields( + JsonFieldInfo[] properties, boolean objectStartFused) { + StringPrefixFields fields = new StringPrefixFields(properties.length); + boolean commaKnown = objectStartFused; + for (int i = 0; i < properties.length; i++) { + JsonFieldInfo property = properties[i]; + if (usesPrefix(property)) { + if (objectStartFused && i == 0) { + if (!canPackStringUtf16Prefix(property, false)) { + fields.name[i] = true; + } + } else { + markStringUtf16PrefixField(property, commaKnown, fields, i); + } + } + if (writeNullFields || property.writeRawType().isPrimitive()) { + commaKnown = true; + } + } + return fields; + } + + private static void markStringUtf16PrefixField( + JsonFieldInfo property, boolean commaKnown, StringPrefixFields fields, int id) { + if (!commaKnown) { + fields.name[id] = true; + fields.comma[id] = true; + return; + } + if (!canPackStringUtf16Prefix(property, true)) { + fields.comma[id] = true; + } + } + + private static final class StringPrefixFields { + private final boolean[] name; + private final boolean[] comma; + + private StringPrefixFields(int size) { + name = new boolean[size]; + comma = new boolean[size]; + } + } + private boolean usesPrefix(JsonFieldInfo property) { JsonFieldKind kind = property.writeKind(); return kind != JsonFieldKind.BOOLEAN && kind != JsonFieldKind.ENUM || writeNullFields && !property.writeRawType().isPrimitive(); } - private Expression writerConstructorExpression(JsonFieldInfo[] properties, boolean utf8) { + private static boolean usesWriteInfo(JsonFieldInfo property) { + JsonFieldKind kind = property.writeKind(); + return kind == JsonFieldKind.BOOLEAN || kind == JsonFieldKind.ENUM; + } + + private Expression writerConstructorExpression( + JsonFieldInfo[] properties, boolean utf8, StringPrefixFields stringPrefixFields) { Expression.ListExpression expressions = new Expression.ListExpression(); Reference propertiesRef = new Reference("properties", TypeRef.of(JsonFieldInfo[].class)); Reference codecsRef = new Reference("codecs", TypeRef.of(JsonCodec[].class)); - expressions.add(new Expression.SuperCall(propertiesRef, codecsRef)); for (int i = 0; i < properties.length; i++) { Expression id = Expression.Literal.ofInt(i); Expression property = new Expression.ArrayValue(propertiesRef, id); - expressions.add( - new Expression.Assign( - new Reference("this.p" + i, TypeRef.of(JsonFieldInfo.class)), property)); + if (usesWriteInfo(properties[i])) { + expressions.add( + new Expression.Assign( + new Reference("this.p" + i, TypeRef.of(JsonFieldInfo.class)), property)); + } if (usesWriteCodec(properties[i])) { Class codecType = codecFieldType(properties[i].writeTypeInfo().codec()); expressions.add( @@ -176,22 +244,40 @@ private Expression writerConstructorExpression(JsonFieldInfo[] properties, boole expressions.add( new Expression.Assign( new Reference("this.u" + i, TypeRef.of(byte[].class)), - new Expression.Invoke(property, "utf8NamePrefix", TypeRef.of(byte[].class)))); + new Expression.Invoke(property, "utf8NamePrefix", TypeRef.of(byte[].class)) + .inline())); expressions.add( new Expression.Assign( new Reference("this.uc" + i, TypeRef.of(byte[].class)), - new Expression.Invoke( - property, "utf8CommaNamePrefix", TypeRef.of(byte[].class)))); + new Expression.Invoke(property, "utf8CommaNamePrefix", TypeRef.of(byte[].class)) + .inline())); } else { expressions.add( new Expression.Assign( new Reference("this.s" + i, TypeRef.of(byte[].class)), - new Expression.Invoke(property, "stringNamePrefix", TypeRef.of(byte[].class)))); + new Expression.Invoke(property, "stringNamePrefix", TypeRef.of(byte[].class)) + .inline())); expressions.add( new Expression.Assign( new Reference("this.sc" + i, TypeRef.of(byte[].class)), - new Expression.Invoke( - property, "stringCommaNamePrefix", TypeRef.of(byte[].class)))); + new Expression.Invoke(property, "stringCommaNamePrefix", TypeRef.of(byte[].class)) + .inline())); + if (stringPrefixFields.name[i]) { + expressions.add( + new Expression.Assign( + new Reference("this.s16" + i, TypeRef.of(byte[].class)), + new Expression.Invoke( + property, "stringUtf16NamePrefix", TypeRef.of(byte[].class)) + .inline())); + } + if (stringPrefixFields.comma[i]) { + expressions.add( + new Expression.Assign( + new Reference("this.sc16" + i, TypeRef.of(byte[].class)), + new Expression.Invoke( + property, "stringUtf16CommaNamePrefix", TypeRef.of(byte[].class)) + .inline())); + } } } } @@ -209,31 +295,95 @@ private Expression writeExpression( "object", new Expression.Cast( new Reference("value", TypeRef.of(Object.class)), TypeRef.of(type))); + Reference writer = writerRef(utf8); + Reference typeResolver = typeResolverRef(); Expression.ListExpression expressions = new Expression.ListExpression(); expressions.add(object); Expression index = null; if (!objectStartFused) { - expressions.add(new Expression.Invoke(writerRef(utf8), "writeObjectStart")); + expressions.add(new Expression.Invoke(writer, "writeObjectStart")); index = new Expression.Variable("index", Expression.Literal.ofInt(0)); expressions.add(index); } boolean commaKnown = objectStartFused; + boolean splitMembers = + properties.length >= (utf8 ? MIN_UTF8_SPLIT_MEMBERS : MIN_STRING_SPLIT_MEMBERS); + List memberGroup = splitMembers ? new ArrayList<>(MEMBER_GROUP_SIZE) : null; for (int i = 0; i < properties.length; i++) { + Expression member; if (objectStartFused && i == 0) { - expressions.add( + member = writeObjectStartPrimitive( - properties[i], builder.fieldValue(properties[i], object), utf8)); + properties[i], builder.fieldValue(properties[i], object), utf8, writer); } else { - expressions.add(writeProp(builder, properties[i], i, utf8, commaKnown, index, object)); + member = + writeProp( + builder, properties[i], i, utf8, commaKnown, index, object, writer, typeResolver); + } + if (splitMembers && commaKnown) { + memberGroup.add(member); + if (memberGroup.size() == MEMBER_GROUP_SIZE) { + addMemberGroup(builder, expressions, memberGroup, object, writer, typeResolver); + } + } else { + if (splitMembers) { + addMemberGroup(builder, expressions, memberGroup, object, writer, typeResolver); + } + expressions.add(member); } if (writeNullFields || properties[i].writeRawType().isPrimitive()) { commaKnown = true; } } - expressions.add(new Expression.Invoke(writerRef(utf8), "writeObjectEnd")); + if (splitMembers) { + addMemberGroup(builder, expressions, memberGroup, object, writer, typeResolver, true); + } + expressions.add(new Expression.Invoke(writer, "writeObjectEnd")); return expressions; } + private static void addMemberGroup( + JsonGeneratedCodecBuilder builder, + Expression.ListExpression expressions, + List memberGroup, + Expression object, + Reference writer, + Reference typeResolver) { + addMemberGroup(builder, expressions, memberGroup, object, writer, typeResolver, false); + } + + private static void addMemberGroup( + JsonGeneratedCodecBuilder builder, + Expression.ListExpression expressions, + List memberGroup, + Expression object, + Reference writer, + Reference typeResolver, + boolean inlineSmallTail) { + if (memberGroup.isEmpty()) { + return; + } + if (memberGroup.size() == 1 || inlineSmallTail && memberGroup.size() <= INLINE_TAIL_MEMBERS) { + for (Expression member : memberGroup) { + expressions.add(member); + } + memberGroup.clear(); + return; + } + LinkedHashSet cutPoints = new LinkedHashSet<>(); + cutPoints.add(object); + cutPoints.add(writer); + cutPoints.add(typeResolver); + expressions.add( + ExpressionOptimizer.invokeGenerated( + builder.context(), + cutPoints, + new Expression.ListExpression(new ArrayList<>(memberGroup)), + "writeMembers", + false)); + memberGroup.clear(); + } + private static boolean canFuseObjectStart(JsonFieldInfo[] properties) { if (properties.length == 0 || !properties[0].writeRawType().isPrimitive()) { return false; @@ -250,16 +400,48 @@ private static boolean canFuseObjectStart(JsonFieldInfo[] properties) { } private static Expression writeObjectStartPrimitive( - JsonFieldInfo property, Expression value, boolean utf8) { + JsonFieldInfo property, Expression value, boolean utf8, Expression writer) { switch (property.writeKind()) { case BYTE: case SHORT: case INT: + if (utf8 && canPackPrefix(property, false)) { + return new Expression.Invoke( + writer, "writeObjectIntField", packedPrefixArgs(property, false, value)); + } + if (!utf8) { + if (canPackStringUtf16Prefix(property, false)) { + return new Expression.Invoke( + writer, "writeObjectIntField", stringPackedPrefixArgs(property, 0, false, value)); + } + return new Expression.Invoke( + writer, + "writeObjectIntField", + prefixRef(false, false, 0), + utf16PrefixRef(false, 0), + value); + } return new Expression.Invoke( - writerRef(utf8), "writeObjectIntField", prefixRef(utf8, false, 0), value); + writer, "writeObjectIntField", prefixRef(utf8, false, 0), value); case LONG: + if (utf8 && canPackPrefix(property, false)) { + return new Expression.Invoke( + writer, "writeObjectLongField", packedPrefixArgs(property, false, value)); + } + if (!utf8) { + if (canPackStringUtf16Prefix(property, false)) { + return new Expression.Invoke( + writer, "writeObjectLongField", stringPackedPrefixArgs(property, 0, false, value)); + } + return new Expression.Invoke( + writer, + "writeObjectLongField", + prefixRef(false, false, 0), + utf16PrefixRef(false, 0), + value); + } return new Expression.Invoke( - writerRef(utf8), "writeObjectLongField", prefixRef(utf8, false, 0), value); + writer, "writeObjectLongField", prefixRef(utf8, false, 0), value); default: throw new ForyJsonException( "Unsupported generated object-start kind " + property.writeKind()); @@ -273,16 +455,17 @@ private Expression writeProp( boolean utf8, boolean commaKnown, Expression index, - Expression object) { + Expression object, + Expression writer, + Expression typeResolver) { Class rawType = property.writeRawType(); if (rawType.isPrimitive()) { return writePrimitive( - property, id, builder.fieldValue(property, object), utf8, commaKnown, index); + property, id, builder.fieldValue(property, object), utf8, commaKnown, index, writer); } Expression value = new Expression.Variable( - "v" + id, - new Expression.Cast(builder.fieldValue(property, object), TypeRef.of(rawType))); + "v" + id, cast(inline(builder.fieldValue(property, object)), TypeRef.of(rawType))); Expression nullValue = new Expression.Null(TypeRef.of(rawType), false); if (writeNullFields) { if (isPrefixValue(property.writeKind())) { @@ -291,24 +474,24 @@ private Expression writeProp( new Expression.If( eq(value, nullValue), new Expression.ListExpression( - writeFieldName(id, utf8, commaKnown, index), - new Expression.Invoke(writerRef(utf8), "writeNull")), - writeValue(property, id, value, utf8, commaKnown, index))); + writeFieldName(property, id, utf8, commaKnown, index, writer), + new Expression.Invoke(writer, "writeNull")), + writeValue(property, id, value, utf8, commaKnown, index, writer, typeResolver))); } return new Expression.ListExpression( value, - writeFieldName(id, utf8, commaKnown, index), + writeFieldName(property, id, utf8, commaKnown, index, writer), new Expression.If( eq(value, nullValue), - new Expression.Invoke(writerRef(utf8), "writeNull"), - writeValue(property, id, value, utf8, true, index))); + new Expression.Invoke(writer, "writeNull"), + writeValue(property, id, value, utf8, true, index, writer, typeResolver))); } Expression write = isPrefixValue(property.writeKind()) - ? writeValue(property, id, value, utf8, commaKnown, index) + ? writeValue(property, id, value, utf8, commaKnown, index, writer, typeResolver) : new Expression.ListExpression( - writeFieldName(id, utf8, commaKnown, index), - writeValue(property, id, value, utf8, true, index)); + writeFieldName(property, id, utf8, commaKnown, index, writer), + writeValue(property, id, value, utf8, true, index, writer, typeResolver)); return new Expression.ListExpression(value, new Expression.If(ne(value, nullValue), write)); } @@ -318,69 +501,168 @@ private Expression writePrimitive( Expression value, boolean utf8, boolean commaKnown, - Expression index) { + Expression index, + Expression writer) { switch (property.writeKind()) { case BOOLEAN: return writeRawFieldValue( - utf8, commaKnown, index, booleanFieldValue(id, value, utf8, commaKnown, index)); + commaKnown, index, booleanFieldValue(id, value, utf8, commaKnown, index), writer); case BYTE: case SHORT: case INT: - return writeNumberField(id, value, false, utf8, commaKnown, index); + return writeNumberField(property, id, value, false, utf8, commaKnown, index, writer); case LONG: - return writeNumberField(id, value, true, utf8, commaKnown, index); + return writeNumberField(property, id, value, true, utf8, commaKnown, index, writer); default: return new Expression.ListExpression( - writeFieldName(id, utf8, commaKnown, index), - writePrimitiveScalar(property.writeKind(), value, utf8)); + writeFieldName(property, id, utf8, commaKnown, index, writer), + writePrimitiveScalar(property.writeKind(), value, writer)); } } private static Expression writeNumberField( + JsonFieldInfo property, int id, Expression value, boolean longValue, boolean utf8, boolean commaKnown, - Expression index) { + Expression index, + Expression writer) { String writerMethod = longValue ? "writeLongField" : "writeIntField"; if (commaKnown) { - return new Expression.Invoke(writerRef(utf8), writerMethod, prefixRef(utf8, true, id), value); + if (utf8 && canPackPrefix(property, true)) { + return new Expression.Invoke(writer, writerMethod, packedPrefixArgs(property, true, value)); + } + if (!utf8) { + if (canPackStringUtf16Prefix(property, true)) { + return new Expression.Invoke( + writer, writerMethod, stringPackedPrefixArgs(property, id, true, value)); + } + return new Expression.Invoke( + writer, writerMethod, prefixRef(false, true, id), utf16PrefixRef(true, id), value); + } + return new Expression.Invoke(writer, writerMethod, prefixRef(utf8, true, id), value); + } + if (utf8 && canPackSinglePrefix(property, false) && canPackSinglePrefix(property, true)) { + return new Expression.ListExpression( + new Expression.Invoke( + writer, writerMethod, packedDynamicPrefixArgs(property, index, value)), + increment(index)); } Expression.ListExpression expressions = new Expression.ListExpression( - new Expression.Invoke( - writerRef(utf8), - writerMethod, - prefixRef(utf8, false, id), - prefixRef(utf8, true, id), - index, - value)); + utf8 + ? new Expression.Invoke( + writer, + writerMethod, + prefixRef(true, false, id), + prefixRef(true, true, id), + index, + value) + : new Expression.Invoke( + writer, + writerMethod, + prefixRef(false, false, id), + prefixRef(false, true, id), + utf16PrefixRef(false, id), + utf16PrefixRef(true, id), + index, + value)); expressions.add(increment(index)); return expressions; } private static Expression writeStringField( - int id, Expression value, boolean utf8, boolean commaKnown, Expression index) { + JsonFieldInfo property, + int id, + Expression value, + boolean utf8, + boolean commaKnown, + Expression index, + Expression writer) { if (commaKnown) { - return new Expression.Invoke( - writerRef(utf8), "writeStringField", prefixRef(utf8, true, id), value); + if (utf8 && canPackPrefix(property, true)) { + return new Expression.Invoke( + writer, "writeStringField", packedPrefixArgs(property, true, value)); + } + if (!utf8) { + if (canPackStringUtf16Prefix(property, true)) { + return new Expression.Invoke( + writer, "writeStringField", stringPackedPrefixArgs(property, id, true, value)); + } + return new Expression.Invoke( + writer, + "writeStringField", + prefixRef(false, true, id), + utf16PrefixRef(true, id), + value); + } + return new Expression.Invoke(writer, "writeStringField", prefixRef(utf8, true, id), value); + } + if (utf8 && canPackSinglePrefix(property, false) && canPackSinglePrefix(property, true)) { + Expression.ListExpression expressions = + new Expression.ListExpression( + new Expression.Invoke( + writer, "writeStringField", packedDynamicPrefixArgs(property, index, value))); + expressions.add(increment(index)); + return expressions; } Expression.ListExpression expressions = new Expression.ListExpression( - new Expression.Invoke( - writerRef(utf8), - "writeStringField", - prefixRef(utf8, false, id), - prefixRef(utf8, true, id), - index, - value)); + utf8 + ? new Expression.Invoke( + writer, + "writeStringField", + prefixRef(true, false, id), + prefixRef(true, true, id), + index, + value) + : new Expression.Invoke( + writer, + "writeStringField", + prefixRef(false, false, id), + prefixRef(false, true, id), + utf16PrefixRef(false, id), + utf16PrefixRef(true, id), + index, + value)); expressions.add(increment(index)); return expressions; } private static Expression writeFieldName( - int id, boolean utf8, boolean commaKnown, Expression index) { + JsonFieldInfo property, + int id, + boolean utf8, + boolean commaKnown, + Expression index, + Expression writer) { + if (commaKnown && utf8 && canPackPrefix(property, true)) { + return new Expression.Invoke(writer, "writeRawValue", packedPrefixArgs(property, true)); + } + if (!utf8 && commaKnown) { + if (canPackStringUtf16Prefix(property, true)) { + return new Expression.Invoke( + writer, "writeRawValue", stringPackedPrefixArgs(property, id, true)); + } + return new Expression.Invoke( + writer, "writeRawValue", prefixRef(false, true, id), utf16PrefixRef(true, id)); + } + if (!utf8) { + Expression.ListExpression expressions = + new Expression.ListExpression( + new Expression.Invoke( + writer, + "writeRawValue", + prefixRef(false, false, id), + prefixRef(false, true, id), + utf16PrefixRef(false, id), + utf16PrefixRef(true, id), + index)); + expressions.add(increment(index)); + return expressions; + } Expression prefix = commaKnown ? prefixRef(utf8, true, id) @@ -391,8 +673,7 @@ private static Expression writeFieldName( true, TypeRef.of(byte[].class)); Expression.ListExpression expressions = - new Expression.ListExpression( - new Expression.Invoke(writerRef(utf8), "writeRawValue", prefix)); + new Expression.ListExpression(new Expression.Invoke(writer, "writeRawValue", prefix)); if (!commaKnown) { expressions.add(increment(index)); } @@ -405,12 +686,13 @@ private Expression writeValue( Expression value, boolean utf8, boolean commaKnown, - Expression index) { + Expression index, + Expression writer, + Expression typeResolver) { JsonFieldKind kind = property.writeKind(); switch (kind) { case BOOLEAN: return writeRawFieldValue( - utf8, commaKnown, index, booleanFieldValue( @@ -418,58 +700,150 @@ private Expression writeValue( new Expression.Invoke(value, "booleanValue", TypeRef.of(boolean.class)).inline(), utf8, commaKnown, - index)); + index), + writer); case BYTE: case SHORT: case INT: return writeNumberField( + property, id, new Expression.Invoke(value, "intValue", TypeRef.of(int.class)).inline(), false, utf8, commaKnown, - index); + index, + writer); case LONG: return writeNumberField( + property, id, new Expression.Invoke(value, "longValue", TypeRef.of(long.class)).inline(), true, utf8, commaKnown, - index); + index, + writer); case STRING: - return writeStringField(id, value, utf8, commaKnown, index); + return writeStringField(property, id, value, utf8, commaKnown, index, writer); case ENUM: return writeRawFieldValue( - utf8, commaKnown, index, enumFieldValue(id, value, utf8, commaKnown, index)); + commaKnown, + index, + enumFieldValue(id, value, utf8, commaKnown, index), + utf8 ? null : stringUtf16EnumFieldValue(id, value, commaKnown, index), + writer); case FLOAT: case DOUBLE: case CHAR: - return writeScalar(kind, value, utf8); + return writeScalar(kind, value, writer); case ARRAY: + Expression array = writeExactUtf8Array(property.writeRawType(), value, utf8, writer); + return array == null ? writeCodec(id, value, utf8, writer, typeResolver) : array; case MAP: - return writeCodec(id, value, utf8); + return writeCodec(id, value, utf8, writer, typeResolver); case COLLECTION: if (property.writeElementRawType() == String.class) { - return writeStringCollection(value, utf8); + return writeStringCollection(value, utf8, writer); } - return writeCodec(id, value, utf8); + return writeCodec(id, value, utf8, writer, typeResolver); + case OBJECT: + Expression scalar = writeExactUtf8Scalar(property.writeRawType(), value, utf8, writer); + return scalar == null ? writeCodec(id, value, utf8, writer, typeResolver) : scalar; default: - return writeCodec(id, value, utf8); + return writeCodec(id, value, utf8, writer, typeResolver); } } private static Expression writeRawFieldValue( - boolean utf8, boolean commaKnown, Expression index, Expression value) { - Expression.ListExpression expressions = - new Expression.ListExpression( - new Expression.Invoke(writerRef(utf8), "writeRawValue", value)); + boolean commaKnown, Expression index, Expression value, Expression writer) { + return writeRawFieldValue(commaKnown, index, value, null, writer); + } + + private static Expression writeRawFieldValue( + boolean commaKnown, + Expression index, + Expression value, + Expression utf16Value, + Expression writer) { + Expression write = + utf16Value == null + ? new Expression.Invoke(writer, "writeRawValue", value) + : new Expression.Invoke(writer, "writeRawValue", value, utf16Value); + Expression.ListExpression expressions = new Expression.ListExpression(write); if (!commaKnown) { expressions.add(increment(index)); } return expressions; } + private static Expression[] packedPrefixArgs( + JsonFieldInfo property, boolean comma, Expression... extraArgs) { + byte[] prefix = comma ? property.utf8CommaNamePrefix() : property.utf8NamePrefix(); + Expression[] args = new Expression[3 + extraArgs.length]; + args[0] = Expression.Literal.ofLong(packedPrefixWord(prefix, 0)); + args[1] = Expression.Literal.ofLong(packedPrefixWord(prefix, Long.BYTES)); + args[2] = Expression.Literal.ofInt(prefix.length); + System.arraycopy(extraArgs, 0, args, 3, extraArgs.length); + return args; + } + + private static Expression[] stringPackedPrefixArgs( + JsonFieldInfo property, int id, boolean comma, Expression... extraArgs) { + byte[] prefix = + comma ? property.stringUtf16CommaNamePrefix() : property.stringUtf16NamePrefix(); + Expression[] args = new Expression[6 + extraArgs.length]; + args[0] = prefixRef(false, comma, id); + args[1] = Expression.Literal.ofLong(packedPrefixWord(prefix, 0)); + args[2] = Expression.Literal.ofLong(packedPrefixWord(prefix, Long.BYTES)); + args[3] = Expression.Literal.ofLong(packedPrefixWord(prefix, Long.BYTES * 2)); + args[4] = Expression.Literal.ofLong(packedPrefixWord(prefix, Long.BYTES * 3)); + args[5] = Expression.Literal.ofInt(prefix.length); + System.arraycopy(extraArgs, 0, args, 6, extraArgs.length); + return args; + } + + private static Expression[] packedDynamicPrefixArgs( + JsonFieldInfo property, Expression index, Expression... extraArgs) { + byte[] namePrefix = property.utf8NamePrefix(); + byte[] commaPrefix = property.utf8CommaNamePrefix(); + Expression[] args = new Expression[5 + extraArgs.length]; + args[0] = Expression.Literal.ofLong(packedPrefixWord(namePrefix, 0)); + args[1] = Expression.Literal.ofLong(packedPrefixWord(commaPrefix, 0)); + args[2] = Expression.Literal.ofInt(namePrefix.length); + args[3] = Expression.Literal.ofInt(commaPrefix.length); + args[4] = index; + System.arraycopy(extraArgs, 0, args, 5, extraArgs.length); + return args; + } + + private static boolean canPackPrefix(JsonFieldInfo property, boolean comma) { + int length = comma ? property.utf8CommaNamePrefix().length : property.utf8NamePrefix().length; + return length <= Long.BYTES * 2; + } + + private static boolean canPackStringUtf16Prefix(JsonFieldInfo property, boolean comma) { + int length = + comma + ? property.stringUtf16CommaNamePrefix().length + : property.stringUtf16NamePrefix().length; + return length <= Long.BYTES * 4; + } + + private static boolean canPackSinglePrefix(JsonFieldInfo property, boolean comma) { + int length = comma ? property.utf8CommaNamePrefix().length : property.utf8NamePrefix().length; + return length <= Long.BYTES; + } + + private static long packedPrefixWord(byte[] prefix, int offset) { + long word = 0; + int end = Math.min(prefix.length, offset + Long.BYTES); + for (int i = offset; i < end; i++) { + word |= (prefix[i] & 0xffL) << ((i - offset) << 3); + } + return word; + } + private static Expression booleanFieldValue( int id, Expression value, boolean utf8, boolean commaKnown, Expression index) { return new Expression.Invoke( @@ -492,42 +866,84 @@ private static Expression enumFieldValue( .inline(); } - private static Expression writeCodec(int id, Expression value, boolean utf8) { + private static Expression stringUtf16EnumFieldValue( + int id, Expression value, boolean commaKnown, Expression index) { + return new Expression.Invoke( + fieldRef("p" + id, JsonFieldInfo.class), + "stringUtf16EnumFieldValue", + TypeRef.of(byte[].class), + value, + commaFlag(commaKnown, index)) + .inline(); + } + + private static Expression writeCodec( + int id, Expression value, boolean utf8, Expression writer, Expression typeResolver) { return new Expression.Invoke( fieldRef("c" + id, JsonCodec.class), utf8 ? "writeUtf8" : "writeString", - writerRef(utf8), + writer, value, - typeResolverRef()); + typeResolver); } - private static Expression writeStringCollection(Expression value, boolean utf8) { - return new Expression.ListExpression( - new Expression.Invoke(writerRef(utf8), "writeArrayStart"), - new Expression.ForEach( - value, - TypeRef.of(String.class), - true, - (index, element) -> - new Expression.Invoke(writerRef(utf8), "writeStringElement", index, element)), - new Expression.Invoke(writerRef(utf8), "writeArrayEnd")); + private static Expression writeExactUtf8Scalar( + Class rawType, Expression value, boolean utf8, Expression writer) { + if (!utf8) { + return null; + } + String writerMethod; + if (rawType == UUID.class) { + writerMethod = "writeUuid"; + } else if (rawType == LocalDate.class) { + writerMethod = "writeLocalDate"; + } else if (rawType == OffsetDateTime.class) { + writerMethod = "writeOffsetDateTime"; + } else if (rawType == BigDecimal.class) { + return new Expression.Invoke( + writer, + "writeNumber", + new Expression.Invoke(value, "toString", TypeRef.of(String.class)).inline()); + } else { + return null; + } + return new Expression.Invoke(writer, writerMethod, value); + } + + private static Expression writeExactUtf8Array( + Class rawType, Expression value, boolean utf8, Expression writer) { + if (!utf8) { + return null; + } + if (rawType == String[].class) { + return new Expression.Invoke(writer, "writeStringArray", value); + } + if (rawType == long[].class) { + return new Expression.Invoke(writer, "writeLongArray", value); + } + return null; } - private Expression writeScalar(JsonFieldKind kind, Expression value, boolean utf8) { + private static Expression writeStringCollection( + Expression value, boolean utf8, Expression writer) { + return new Expression.Invoke(writer, "writeStringCollection", value); + } + + private Expression writeScalar(JsonFieldKind kind, Expression value, Expression writer) { switch (kind) { case FLOAT: return new Expression.Invoke( - writerRef(utf8), + writer, "writeFloat", new Expression.Invoke(value, "floatValue", TypeRef.of(float.class)).inline()); case DOUBLE: return new Expression.Invoke( - writerRef(utf8), + writer, "writeDouble", new Expression.Invoke(value, "doubleValue", TypeRef.of(double.class)).inline()); case CHAR: return new Expression.Invoke( - writerRef(utf8), + writer, "writeChar", new Expression.Invoke(value, "charValue", TypeRef.of(char.class)).inline()); default: @@ -535,14 +951,14 @@ private Expression writeScalar(JsonFieldKind kind, Expression value, boolean utf } } - private Expression writePrimitiveScalar(JsonFieldKind kind, Expression value, boolean utf8) { + private Expression writePrimitiveScalar(JsonFieldKind kind, Expression value, Expression writer) { switch (kind) { case FLOAT: - return new Expression.Invoke(writerRef(utf8), "writeFloat", value); + return new Expression.Invoke(writer, "writeFloat", value); case DOUBLE: - return new Expression.Invoke(writerRef(utf8), "writeDouble", value); + return new Expression.Invoke(writer, "writeDouble", value); case CHAR: - return new Expression.Invoke(writerRef(utf8), "writeChar", value); + return new Expression.Invoke(writer, "writeChar", value); default: throw new ForyJsonException("Unsupported generated primitive kind " + kind); } @@ -558,12 +974,16 @@ private static Reference prefixRef(boolean utf8, boolean comma, int id) { return fieldRef(prefix + id, byte[].class); } + private static Reference utf16PrefixRef(boolean comma, int id) { + return fieldRef((comma ? "sc16" : "s16") + id, byte[].class); + } + private static Expression commaFlag(boolean commaKnown, Expression index) { return commaKnown ? Expression.Literal.True : ne(index, Expression.Literal.ofInt(0)); } private static Expression increment(Expression value) { - return new Expression.Assign(value, new Expression.Add(value, Expression.Literal.ofInt(1))); + return new Expression.Assign(value, add(value, Expression.Literal.ofInt(1))); } private static boolean isPrefixValue(JsonFieldKind kind) { diff --git a/java/fory-json/src/main/java/org/apache/fory/json/meta/JsonAsciiToken.java b/java/fory-json/src/main/java/org/apache/fory/json/meta/JsonAsciiToken.java index 44b504e865..16285ff41e 100644 --- a/java/fory-json/src/main/java/org/apache/fory/json/meta/JsonAsciiToken.java +++ b/java/fory-json/src/main/java/org/apache/fory/json/meta/JsonAsciiToken.java @@ -21,6 +21,7 @@ public final class JsonAsciiToken { private static final int MAX_SUFFIX_LENGTH = 3; + private static final int MAX_LONG_SUFFIX_LENGTH = Long.BYTES; private JsonAsciiToken() {} @@ -29,6 +30,19 @@ public static boolean isPackable(String token) { if (length == 0 || suffixLength(length) > MAX_SUFFIX_LENGTH) { return false; } + return isLatin1Token(token); + } + + public static boolean isLongPackable(String token) { + int length = token.length(); + if (length == 0 || suffixLength(length) > MAX_LONG_SUFFIX_LENGTH) { + return false; + } + return isLatin1Token(token); + } + + private static boolean isLatin1Token(String token) { + int length = token.length(); for (int i = 0; i < length; i++) { char ch = token.charAt(i); if (ch == 0 || ch > 0xFF) { @@ -61,6 +75,20 @@ public static int suffix(String token) { return value; } + public static long suffixLong(String token) { + int suffixLength = suffixLength(token.length()); + long value = 0; + for (int i = 0; i < suffixLength; i++) { + value |= (long) (token.charAt(i + Long.BYTES) & 0xFF) << (i << 3); + } + return value; + } + + public static long suffixMask(int tokenLength) { + int suffixLength = suffixLength(tokenLength); + return suffixLength == Long.BYTES ? -1L : (1L << (suffixLength << 3)) - 1; + } + public static int suffixLength(int tokenLength) { return Math.max(0, tokenLength - Long.BYTES); } diff --git a/java/fory-json/src/main/java/org/apache/fory/json/meta/JsonFieldAccessor.java b/java/fory-json/src/main/java/org/apache/fory/json/meta/JsonFieldAccessor.java index 5bed687f76..712d58d233 100644 --- a/java/fory-json/src/main/java/org/apache/fory/json/meta/JsonFieldAccessor.java +++ b/java/fory-json/src/main/java/org/apache/fory/json/meta/JsonFieldAccessor.java @@ -19,7 +19,13 @@ package org.apache.fory.json.meta; +import java.lang.invoke.MethodHandle; import java.lang.reflect.Field; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import org.apache.fory.json.ForyJsonException; +import org.apache.fory.platform.AndroidSupport; +import org.apache.fory.platform.internal._JDKAccess; import org.apache.fory.reflect.FieldAccessor; public abstract class JsonFieldAccessor { @@ -27,9 +33,21 @@ public Object getObject(Object target) { throw new UnsupportedOperationException(); } - public abstract Field field(); + public Field field() { + return null; + } - public abstract FieldAccessor coreAccessor(); + public Method getter() { + return null; + } + + public Method setter() { + return null; + } + + public FieldAccessor coreAccessor() { + return null; + } public boolean getBoolean(Object target) { return (Boolean) getObject(target); @@ -103,6 +121,14 @@ public static JsonFieldAccessor forField(Field field) { return new FieldJsonAccessor(FieldAccessor.createAccessor(field)); } + public static JsonFieldAccessor forGetter(Method getter) { + return new GetterJsonAccessor(getter); + } + + public static JsonFieldAccessor forSetter(Method setter) { + return new SetterJsonAccessor(setter); + } + private static final class FieldJsonAccessor extends JsonFieldAccessor { private final FieldAccessor accessor; @@ -210,4 +236,83 @@ public void putChar(Object target, char value) { accessor.putChar(target, value); } } + + private static final class GetterJsonAccessor extends JsonFieldAccessor { + private final Method getter; + private final MethodHandle getterHandle; + + private GetterJsonAccessor(Method getter) { + this.getter = getter; + if (AndroidSupport.IS_ANDROID) { + getter.setAccessible(true); + getterHandle = null; + } else { + getterHandle = methodHandle(getter); + } + } + + @Override + public Method getter() { + return getter; + } + + @Override + public Object getObject(Object target) { + try { + if (AndroidSupport.IS_ANDROID) { + return getter.invoke(target); + } + return getterHandle.invoke(target); + } catch (Throwable e) { + throw accessException(getter, e); + } + } + } + + private static final class SetterJsonAccessor extends JsonFieldAccessor { + private final Method setter; + private final MethodHandle setterHandle; + + private SetterJsonAccessor(Method setter) { + this.setter = setter; + if (AndroidSupport.IS_ANDROID) { + setter.setAccessible(true); + setterHandle = null; + } else { + setterHandle = methodHandle(setter); + } + } + + @Override + public Method setter() { + return setter; + } + + @Override + public void putObject(Object target, Object value) { + try { + if (AndroidSupport.IS_ANDROID) { + setter.invoke(target, value); + } else { + setterHandle.invoke(target, value); + } + } catch (Throwable e) { + throw accessException(setter, e); + } + } + } + + private static MethodHandle methodHandle(Method method) { + try { + return _JDKAccess._trustedLookup(method.getDeclaringClass()).unreflect(method); + } catch (IllegalAccessException e) { + throw accessException(method, e); + } + } + + private static ForyJsonException accessException(Method method, Throwable e) { + Throwable cause = + e instanceof InvocationTargetException ? ((InvocationTargetException) e).getCause() : e; + return new ForyJsonException("Cannot access JSON property method " + method, cause); + } } diff --git a/java/fory-json/src/main/java/org/apache/fory/json/meta/JsonFieldInfo.java b/java/fory-json/src/main/java/org/apache/fory/json/meta/JsonFieldInfo.java index 01a31a374a..b3b706e658 100644 --- a/java/fory-json/src/main/java/org/apache/fory/json/meta/JsonFieldInfo.java +++ b/java/fory-json/src/main/java/org/apache/fory/json/meta/JsonFieldInfo.java @@ -20,6 +20,7 @@ package org.apache.fory.json.meta; import java.lang.reflect.Field; +import java.lang.reflect.Method; import java.lang.reflect.Type; import java.nio.charset.StandardCharsets; import java.util.Collection; @@ -33,6 +34,7 @@ import org.apache.fory.json.writer.JsonWriter; import org.apache.fory.json.writer.StringJsonWriter; import org.apache.fory.json.writer.Utf8JsonWriter; +import org.apache.fory.memory.NativeByteOrder; public final class JsonFieldInfo { private static final int KIND_BOOLEAN = 1; @@ -54,6 +56,9 @@ public final class JsonFieldInfo { private final String name; private final Field writeField; + private final Method writeGetter; + private final Field readField; + private final Method readSetter; private final Type writeType; private final Class writeRawType; private final Type readType; @@ -71,12 +76,16 @@ public final class JsonFieldInfo { private final Class readElementRawType; private final byte[] stringNamePrefix; private final byte[] stringCommaNamePrefix; + private final byte[] stringUtf16NamePrefix; + private final byte[] stringUtf16CommaNamePrefix; private final byte[] utf8NamePrefix; private final byte[] utf8CommaNamePrefix; private final byte[][] stringEnumValues; private final byte[][] stringElementEnumValues; private final byte[][] stringEnumNameValues; private final byte[][] stringEnumCommaValues; + private final byte[][] stringUtf16EnumNameValues; + private final byte[][] stringUtf16EnumCommaValues; private final byte[][] utf8EnumValues; private final byte[][] utf8ElementEnumValues; private final byte[][] utf8EnumNameValues; @@ -85,6 +94,10 @@ public final class JsonFieldInfo { private final byte[] stringTrueCommaToken; private final byte[] stringFalseNameToken; private final byte[] stringFalseCommaToken; + private final byte[] stringUtf16TrueNameToken; + private final byte[] stringUtf16TrueCommaToken; + private final byte[] stringUtf16FalseNameToken; + private final byte[] stringUtf16FalseCommaToken; private final byte[] utf8TrueNameToken; private final byte[] utf8TrueCommaToken; private final byte[] utf8FalseNameToken; @@ -98,16 +111,21 @@ public final class JsonFieldInfo { public JsonFieldInfo( String name, Field writeField, + Method writeGetter, Field readField, + Method readSetter, JsonFieldAccessor writeAccessor, JsonFieldAccessor readAccessor) { this.name = name; nameHash = JsonFieldNameHash.hash(name); this.writeField = writeField; - this.writeType = fieldType(writeField); - this.writeRawType = fieldRawType(writeField); - this.readType = fieldType(readField); - this.readRawType = fieldRawType(readField); + this.writeGetter = writeGetter; + this.readField = readField; + this.readSetter = readSetter; + this.writeType = writeType(writeField, writeGetter); + this.writeRawType = writeRawType(writeField, writeGetter); + this.readType = readType(readField, readSetter); + this.readRawType = readRawType(readField, readSetter); this.writeAccessor = writeAccessor; this.readAccessor = readAccessor; writeKind = writeRawType == null ? null : kind(writeRawType); @@ -126,6 +144,8 @@ public JsonFieldInfo( String utf8Prefix = JsonStringEscaper.escapedNamePrefix(name, false); stringNamePrefix = stringPrefix.getBytes(StandardCharsets.ISO_8859_1); stringCommaNamePrefix = ("," + stringPrefix).getBytes(StandardCharsets.ISO_8859_1); + stringUtf16NamePrefix = toUtf16Bytes(stringNamePrefix); + stringUtf16CommaNamePrefix = toUtf16Bytes(stringCommaNamePrefix); utf8NamePrefix = utf8Prefix.getBytes(StandardCharsets.UTF_8); utf8CommaNamePrefix = ("," + utf8Prefix).getBytes(StandardCharsets.UTF_8); stringEnumValues = writeKind == JsonFieldKind.ENUM ? stringEnumValues(writeRawType) : null; @@ -135,6 +155,14 @@ public JsonFieldInfo( writeKind == JsonFieldKind.ENUM ? fieldValues(stringCommaNamePrefix, stringEnumValues) : null; + stringUtf16EnumNameValues = + writeKind == JsonFieldKind.ENUM + ? utf16FieldValues(stringUtf16NamePrefix, stringEnumValues) + : null; + stringUtf16EnumCommaValues = + writeKind == JsonFieldKind.ENUM + ? utf16FieldValues(stringUtf16CommaNamePrefix, stringEnumValues) + : null; stringElementEnumValues = writeElementRawType != null && writeElementRawType.isEnum() ? stringEnumValues(writeElementRawType) @@ -153,6 +181,10 @@ public JsonFieldInfo( stringTrueCommaToken = join(stringCommaNamePrefix, TRUE_BYTES); stringFalseNameToken = join(stringNamePrefix, FALSE_BYTES); stringFalseCommaToken = join(stringCommaNamePrefix, FALSE_BYTES); + stringUtf16TrueNameToken = join(stringUtf16NamePrefix, toUtf16Bytes(TRUE_BYTES)); + stringUtf16TrueCommaToken = join(stringUtf16CommaNamePrefix, toUtf16Bytes(TRUE_BYTES)); + stringUtf16FalseNameToken = join(stringUtf16NamePrefix, toUtf16Bytes(FALSE_BYTES)); + stringUtf16FalseCommaToken = join(stringUtf16CommaNamePrefix, toUtf16Bytes(FALSE_BYTES)); utf8TrueNameToken = join(utf8NamePrefix, TRUE_BYTES); utf8TrueCommaToken = join(utf8CommaNamePrefix, TRUE_BYTES); utf8FalseNameToken = join(utf8NamePrefix, FALSE_BYTES); @@ -162,6 +194,10 @@ public JsonFieldInfo( stringTrueCommaToken = null; stringFalseNameToken = null; stringFalseCommaToken = null; + stringUtf16TrueNameToken = null; + stringUtf16TrueCommaToken = null; + stringUtf16FalseNameToken = null; + stringUtf16FalseCommaToken = null; utf8TrueNameToken = null; utf8TrueCommaToken = null; utf8FalseNameToken = null; @@ -181,14 +217,26 @@ public Field writeField() { return writeField; } + public Method writeGetter() { + return writeGetter; + } + public Type writeType() { return writeType; } + private static Type writeType(Field field, Method getter) { + return getter == null ? fieldType(field) : getter.getGenericReturnType(); + } + public Class writeRawType() { return writeRawType; } + private static Class writeRawType(Field field, Method getter) { + return getter == null ? fieldRawType(field) : getter.getReturnType(); + } + public JsonFieldKind writeKind() { return writeKind; } @@ -225,14 +273,26 @@ public Type readType() { return readType; } + private static Type readType(Field field, Method setter) { + return setter == null ? fieldType(field) : setter.getGenericParameterTypes()[0]; + } + public Field readField() { - return readAccessor == null ? null : readAccessor.field(); + return readField; + } + + public Method readSetter() { + return readSetter; } public Class readRawType() { return readRawType; } + private static Class readRawType(Field field, Method setter) { + return setter == null ? fieldRawType(field) : setter.getParameterTypes()[0]; + } + public JsonFieldKind readKind() { return readKind; } @@ -297,6 +357,14 @@ public byte[] stringCommaNamePrefix() { return stringCommaNamePrefix; } + public byte[] stringUtf16NamePrefix() { + return stringUtf16NamePrefix; + } + + public byte[] stringUtf16CommaNamePrefix() { + return stringUtf16CommaNamePrefix; + } + public byte[] utf8NamePrefix() { return utf8NamePrefix; } @@ -327,12 +395,22 @@ public byte[] stringEnumFieldValue(Enum value, boolean comma) { return (comma ? stringEnumCommaValues : stringEnumNameValues)[value.ordinal()]; } + public byte[] stringUtf16EnumFieldValue(Enum value, boolean comma) { + return (comma ? stringUtf16EnumCommaValues : stringUtf16EnumNameValues)[value.ordinal()]; + } + public byte[] stringBooleanFieldValue(boolean value, boolean comma) { return value ? (comma ? stringTrueCommaToken : stringTrueNameToken) : (comma ? stringFalseCommaToken : stringFalseNameToken); } + public byte[] stringUtf16BooleanFieldValue(boolean value, boolean comma) { + return value + ? (comma ? stringUtf16TrueCommaToken : stringUtf16TrueNameToken) + : (comma ? stringUtf16FalseCommaToken : stringUtf16FalseNameToken); + } + public byte[] utf8ElementEnumValue(Enum value) { return utf8ElementEnumValues[value.ordinal()]; } @@ -431,7 +509,10 @@ public boolean writeString( if (!writeRawType.isPrimitive()) { return writeStringScalar(writer, object, index); } - writer.writeRawValue(stringBooleanFieldValue(writeAccessor.getBoolean(object), index != 0)); + boolean booleanValue = writeAccessor.getBoolean(object); + writer.writeRawValue( + stringBooleanFieldValue(booleanValue, index != 0), + stringUtf16BooleanFieldValue(booleanValue, index != 0)); return true; case KIND_BYTE: if (!writeRawType.isPrimitive()) { @@ -624,7 +705,10 @@ private boolean writeStringScalar(StringJsonWriter writer, Object object, int in } switch (writeKind) { case BOOLEAN: - writer.writeRawValue(stringBooleanFieldValue(((Boolean) value).booleanValue(), index != 0)); + boolean booleanValue = ((Boolean) value).booleanValue(); + writer.writeRawValue( + stringBooleanFieldValue(booleanValue, index != 0), + stringUtf16BooleanFieldValue(booleanValue, index != 0)); return true; case BYTE: writer.writeIntField( @@ -708,7 +792,8 @@ private boolean writeStringEnum(StringJsonWriter writer, Object object, int inde writer.writeFieldName(this, index); writer.writeNull(); } else { - writer.writeRawValue(stringEnumFieldValue(value, index != 0)); + writer.writeRawValue( + stringEnumFieldValue(value, index != 0), stringUtf16EnumFieldValue(value, index != 0)); } return true; } @@ -1101,10 +1186,32 @@ private static byte[][] fieldValues(byte[] prefix, byte[][] values) { return fieldValues; } + private static byte[][] utf16FieldValues(byte[] utf16Prefix, byte[][] values) { + byte[][] fieldValues = new byte[values.length][]; + for (int i = 0; i < values.length; i++) { + fieldValues[i] = join(utf16Prefix, toUtf16Bytes(values[i])); + } + return fieldValues; + } + private static byte[] join(byte[] prefix, byte[] token) { byte[] joined = new byte[prefix.length + token.length]; System.arraycopy(prefix, 0, joined, 0, prefix.length); System.arraycopy(token, 0, joined, prefix.length, token.length); return joined; } + + private static byte[] toUtf16Bytes(byte[] latin1) { + byte[] utf16 = new byte[latin1.length << 1]; + if (NativeByteOrder.IS_LITTLE_ENDIAN) { + for (int i = 0, j = 0; i < latin1.length; i++, j += 2) { + utf16[j] = latin1[i]; + } + } else { + for (int i = 0, j = 0; i < latin1.length; i++, j += 2) { + utf16[j + 1] = latin1[i]; + } + } + return utf16; + } } diff --git a/java/fory-json/src/main/java/org/apache/fory/json/reader/JsonReader.java b/java/fory-json/src/main/java/org/apache/fory/json/reader/JsonReader.java index 1cb3552200..75e1bc838a 100644 --- a/java/fory-json/src/main/java/org/apache/fory/json/reader/JsonReader.java +++ b/java/fory-json/src/main/java/org/apache/fory/json/reader/JsonReader.java @@ -48,11 +48,15 @@ public final void clearDepth() { public final void enterDepth() { int nextDepth = depth + 1; if (nextDepth > maxDepth) { - throw error("JSON max depth " + maxDepth + " exceeded"); + throwMaxDepthExceeded(); } depth = nextDepth; } + private void throwMaxDepthExceeded() { + throw error("JSON max depth " + maxDepth + " exceeded"); + } + public final void exitDepth() { depth--; } @@ -281,6 +285,10 @@ public final long readLong() { return negative ? result : -result; } + public double readDouble() { + return Double.parseDouble(readNumber()); + } + public int readFieldNameInt() { try { return Integer.parseInt(readString()); @@ -436,40 +444,6 @@ protected final ForyJsonException error(String message) { return new ForyJsonException(message + " at JSON position " + position); } - protected final void appendEscape(StringBuilder builder) { - if (position >= length()) { - throw error("Unterminated escape"); - } - char escaped = charAt(position++); - switch (escaped) { - case '"': - case '\\': - case '/': - builder.append(escaped); - return; - case 'b': - builder.append('\b'); - return; - case 'f': - builder.append('\f'); - return; - case 'n': - builder.append('\n'); - return; - case 'r': - builder.append('\r'); - return; - case 't': - builder.append('\t'); - return; - case 'u': - appendUnicodeEscape(builder); - return; - default: - throw error("Invalid escape"); - } - } - private void skipObject() { enterDepth(); expect('{'); @@ -575,26 +549,6 @@ private void rejectFractionOrExponent() { } } - private void appendUnicodeEscape(StringBuilder builder) { - char ch = readUnicodeEscape(); - if (Character.isHighSurrogate(ch)) { - if (position + 2 > length() || charAt(position) != '\\' || charAt(position + 1) != 'u') { - throw error("Unpaired high surrogate escape"); - } - position += 2; - char low = readUnicodeEscape(); - if (!Character.isLowSurrogate(low)) { - throw error("Unpaired high surrogate escape"); - } - builder.append(ch); - builder.append(low); - } else if (Character.isLowSurrogate(ch)) { - throw error("Unpaired low surrogate escape"); - } else { - builder.append(ch); - } - } - protected final char readEscapedFieldNameChar() { if (position >= length()) { throw error("Unterminated escape"); diff --git a/java/fory-json/src/main/java/org/apache/fory/json/reader/Latin1JsonReader.java b/java/fory-json/src/main/java/org/apache/fory/json/reader/Latin1JsonReader.java index 8a9ae15696..4eba668bd3 100644 --- a/java/fory-json/src/main/java/org/apache/fory/json/reader/Latin1JsonReader.java +++ b/java/fory-json/src/main/java/org/apache/fory/json/reader/Latin1JsonReader.java @@ -19,14 +19,24 @@ package org.apache.fory.json.reader; +import java.math.BigDecimal; +import java.time.LocalDate; +import java.time.OffsetDateTime; +import java.time.ZoneOffset; +import java.util.Arrays; +import java.util.UUID; import org.apache.fory.json.meta.JsonFieldInfo; import org.apache.fory.json.meta.JsonFieldNameHash; import org.apache.fory.json.meta.JsonFieldTable; import org.apache.fory.memory.LittleEndian; +import org.apache.fory.memory.NativeByteOrder; import org.apache.fory.serializer.StringSerializer; public final class Latin1JsonReader extends JsonReader { private static final byte[] EMPTY_BYTES = new byte[0]; + private static final int INITIAL_STRING_DECODE_BUFFER_SIZE = 1024; + private static final int RETAINED_STRING_DECODE_BUFFER_SIZE = 8192; + private static final boolean LITTLE_ENDIAN = NativeByteOrder.IS_LITTLE_ENDIAN; private static final long BYTE_ONES = 0x0101010101010101L; private static final int INT_BYTE_ONES = 0x01010101; private static final long BYTE_HIGH_BITS = 0x8080808080808080L; @@ -37,19 +47,40 @@ public final class Latin1JsonReader extends JsonReader { private static final int INT_CONTROL_LIMIT_BYTES = 0x20202020; private static final long QUOTE_BYTES = 0x2222222222222222L; private static final int INT_QUOTE_BYTES = 0x22222222; + private static final long LONG_MAX_DIV_10 = Long.MAX_VALUE / 10; + private static final int LONG_MAX_MOD_10 = (int) (Long.MAX_VALUE % 10); + private static final long LONG_MAX_DIV_100 = Long.MAX_VALUE / 100; + private static final int LONG_MAX_MOD_100 = (int) (Long.MAX_VALUE % 100); + private static final long LONG_MIN_DIV_10 = Long.MIN_VALUE / 10; + private static final int LONG_MIN_LAST_DIGIT = (int) -(Long.MIN_VALUE % 10); + private static final long EIGHT_DIGITS = 100_000_000L; + private static final long ASCII_ZEROES = 0x3030_3030_3030_3030L; + private static final long ASCII_NINES = 0x3939_3939_3939_3939L; + private static final long ASCII_HIGH_BITS = 0x8080_8080_8080_8080L; // JSON syntax bytes are ASCII, so hot token checks can compare signed bytes directly. // Latin1 string content and field-name hashing must keep unsigned byte conversion. private byte[] input; + private byte[] stringDecodeBuffer = new byte[INITIAL_STRING_DECODE_BUFFER_SIZE]; public Latin1JsonReader() { input = EMPTY_BYTES; } + public Latin1JsonReader(byte[] input) { + this.input = input; + } + public Latin1JsonReader(String input) { reset(input); } + public Latin1JsonReader reset(byte[] input) { + this.input = input; + position = 0; + return this; + } + public Latin1JsonReader reset(String input) { if (!StringSerializer.isBytesBackedString()) { throw new IllegalStateException("Latin1JsonReader requires byte-backed strings"); @@ -66,6 +97,9 @@ public Latin1JsonReader reset(String input) { public void clear() { input = EMPTY_BYTES; position = 0; + if (stringDecodeBuffer.length > RETAINED_STRING_DECODE_BUFFER_SIZE) { + stringDecodeBuffer = new byte[RETAINED_STRING_DECODE_BUFFER_SIZE]; + } } public boolean consumeToken(char expected) { @@ -384,6 +418,16 @@ private long readLongToken() { if (safeEnd > inputLength) { safeEnd = inputLength; } + int block = parseEightDigits(bytes, offset, safeEnd); + if (block >= 0) { + result = result * EIGHT_DIGITS + block; + offset += 8; + block = parseEightDigits(bytes, offset, safeEnd); + if (block >= 0) { + result = result * EIGHT_DIGITS + block; + offset += 8; + } + } while (offset < safeEnd) { ch = bytes[offset]; if (ch < '0' || ch > '9') { @@ -410,7 +454,7 @@ private long readPositiveLongTail(byte[] bytes, int offset, int inputLength, lon break; } int digit = ch - '0'; - if (result > (Long.MAX_VALUE - digit) / 10) { + if (result > LONG_MAX_DIV_10 || (result == LONG_MAX_DIV_10 && digit > LONG_MAX_MOD_10)) { position = offset; throw error("Long overflow"); } @@ -423,15 +467,15 @@ private long readPositiveLongTail(byte[] bytes, int offset, int inputLength, lon } private long readNegativeLongToken(int start) { - position = start + 1; - long result = 0; - long limit = Long.MIN_VALUE; - if (position >= input.length) { + byte[] bytes = input; + int offset = start + 1; + int inputLength = bytes.length; + if (offset >= inputLength) { throw error("Expected digit"); } - int ch = input[position]; + int ch = bytes[offset]; if (ch == '0') { - position++; + position = offset + 1; rejectLeadingDigitFast(); rejectFractionOrExponentFast(); return 0; @@ -439,27 +483,495 @@ private long readNegativeLongToken(int start) { if (ch < '1' || ch > '9') { throw error("Expected digit"); } - long multmin = limit / 10; - while (position < input.length) { - ch = input[position]; + long result = '0' - ch; + offset++; + int safeEnd = offset + 17; + if (safeEnd > inputLength) { + safeEnd = inputLength; + } + int block = parseEightDigits(bytes, offset, safeEnd); + if (block >= 0) { + result = result * EIGHT_DIGITS - block; + offset += 8; + block = parseEightDigits(bytes, offset, safeEnd); + if (block >= 0) { + result = result * EIGHT_DIGITS - block; + offset += 8; + } + } + while (offset < safeEnd) { + ch = bytes[offset]; if (ch < '0' || ch > '9') { break; } - int digit = ch - '0'; - if (result < multmin) { - throw error("Long overflow"); + result = result * 10 - (ch - '0'); + offset++; + } + if (offset < inputLength) { + ch = bytes[offset]; + if (ch >= '0' && ch <= '9') { + return readNegativeLongTail(bytes, offset, inputLength, result); } - result *= 10; - if (result < Long.MIN_VALUE + digit) { + } + position = offset; + rejectFractionOrExponentFast(); + return result; + } + + private long readNegativeLongTail(byte[] bytes, int offset, int inputLength, long result) { + while (offset < inputLength) { + int ch = bytes[offset]; + if (ch < '0' || ch > '9') { + break; + } + int digit = ch - '0'; + if (result < LONG_MIN_DIV_10 || (result == LONG_MIN_DIV_10 && digit > LONG_MIN_LAST_DIGIT)) { + position = offset; throw error("Long overflow"); } - result -= digit; - position++; + result = result * 10 - digit; + offset++; } + position = offset; rejectFractionOrExponentFast(); return result; } + private static int parseEightDigits(byte[] bytes, int offset, int safeEnd) { + if (offset + 8 > safeEnd) { + return -1; + } + // Keep the compact byte-lane arithmetic local to the byte-backed readers so generated parse + // call sites can inline long tokens without eight separate digit checks. + long chunk = LittleEndian.getInt64(bytes, offset); + long digits = chunk - ASCII_ZEROES; + if (((digits | (ASCII_NINES - chunk)) & ASCII_HIGH_BITS) != 0) { + return -1; + } + long pairs = (digits * 10 + (digits >>> 8)) & 0x00FF_00FF_00FF_00FFL; + long quads = (pairs * 100 + (pairs >>> 16)) & 0x0000_FFFF_0000_FFFFL; + return (int) ((quads & 0xFFFF) * 10_000 + (quads >>> 32)); + } + + public BigDecimal readBigDecimal() { + skipWhitespaceFast(); + return readBigDecimalToken(); + } + + public UUID readUuid() { + skipWhitespaceFast(); + int mark = position; + try { + return readUuidToken(); + } catch (RuntimeException e) { + position = mark; + return UUID.fromString(readStringToken()); + } + } + + @Override + public double readDouble() { + skipWhitespaceFast(); + return readDoubleToken(); + } + + public double readNextDoubleValue() { + if (position < input.length) { + int ch = input[position]; + if (ch > ' ' || !isWhitespace(ch)) { + return readDoubleToken(); + } + } + return readDouble(); + } + + public double readDoubleTokenValue() { + return readDoubleToken(); + } + + private BigDecimal readBigDecimalToken() { + byte[] bytes = input; + int offset = position; + int start = offset; + int inputLength = bytes.length; + if (offset >= inputLength) { + return readBigDecimalFallback(start); + } + int ch = bytes[offset]; + if (ch == '-') { + return readSignedBigDecimalToken(start); + } + long unscaled = 0; + int scale = 0; + if (ch == '0') { + offset++; + position = offset; + rejectLeadingDigitFast(); + } else if (ch >= '1' && ch <= '9') { + do { + int digit = ch - '0'; + if (unscaled > (Long.MAX_VALUE - digit) / 10) { + return readBigDecimalFallback(start); + } + unscaled = unscaled * 10 + digit; + offset++; + if (offset >= inputLength) { + break; + } + ch = bytes[offset]; + } while (ch >= '0' && ch <= '9'); + } else { + return readBigDecimalFallback(start); + } + if (offset < inputLength && bytes[offset] == '.') { + offset++; + int fractionStart = offset; + while (offset < inputLength) { + ch = bytes[offset]; + if (ch < '0' || ch > '9') { + break; + } + int digit = ch - '0'; + if (unscaled > (Long.MAX_VALUE - digit) / 10) { + return readBigDecimalFallback(start); + } + unscaled = unscaled * 10 + digit; + scale++; + offset++; + } + if (offset == fractionStart) { + return readBigDecimalFallback(start); + } + } + if (offset < inputLength) { + ch = bytes[offset]; + if (ch == 'e' || ch == 'E') { + return readBigDecimalFallback(start); + } + } + position = offset; + return BigDecimal.valueOf(unscaled, scale); + } + + private BigDecimal readSignedBigDecimalToken(int start) { + byte[] bytes = input; + int offset = start + 1; + int inputLength = bytes.length; + if (offset >= inputLength) { + return readBigDecimalFallback(start); + } + int ch = bytes[offset]; + long unscaled = 0; + int scale = 0; + if (ch == '0') { + offset++; + position = offset; + rejectLeadingDigitFast(); + } else if (ch >= '1' && ch <= '9') { + do { + int digit = ch - '0'; + if (unscaled > (Long.MAX_VALUE - digit) / 10) { + return readBigDecimalFallback(start); + } + unscaled = unscaled * 10 + digit; + offset++; + if (offset >= inputLength) { + break; + } + ch = bytes[offset]; + } while (ch >= '0' && ch <= '9'); + } else { + return readBigDecimalFallback(start); + } + if (offset < inputLength && bytes[offset] == '.') { + offset++; + int fractionStart = offset; + while (offset < inputLength) { + ch = bytes[offset]; + if (ch < '0' || ch > '9') { + break; + } + int digit = ch - '0'; + if (unscaled > (Long.MAX_VALUE - digit) / 10) { + return readBigDecimalFallback(start); + } + unscaled = unscaled * 10 + digit; + scale++; + offset++; + } + if (offset == fractionStart) { + return readBigDecimalFallback(start); + } + } + if (offset < inputLength) { + ch = bytes[offset]; + if (ch == 'e' || ch == 'E') { + return readBigDecimalFallback(start); + } + } + position = offset; + return BigDecimal.valueOf(-unscaled, scale); + } + + private BigDecimal readBigDecimalFallback(int start) { + position = start; + return new BigDecimal(readNumber()); + } + + private UUID readUuidToken() { + byte[] bytes = input; + int offset = position; + int start = offset + 1; + if (offset + 38 > bytes.length || bytes[offset] != '"') { + throw new IllegalArgumentException(); + } + if (bytes[start + 8] != '-' + || bytes[start + 13] != '-' + || bytes[start + 18] != '-' + || bytes[start + 23] != '-' + || bytes[start + 36] != '"') { + throw new IllegalArgumentException(); + } + long msb = parseHex(bytes, start, 8); + msb = (msb << 16) | parseHex(bytes, start + 9, 4); + msb = (msb << 16) | parseHex(bytes, start + 14, 4); + long lsb = parseHex(bytes, start + 19, 4); + lsb = (lsb << 48) | parseHex(bytes, start + 24, 12); + position = start + 37; + return new UUID(msb, lsb); + } + + private static long parseHex(byte[] bytes, int offset, int length) { + long value = 0; + for (int i = 0; i < length; i++) { + value = (value << 4) | hexValue(bytes[offset + i]); + } + return value; + } + + private static int hexValue(int ch) { + if (ch >= '0' && ch <= '9') { + return ch - '0'; + } + int lower = ch | 0x20; + if (lower >= 'a' && lower <= 'f') { + return lower - 'a' + 10; + } + throw new IllegalArgumentException(); + } + + private double readDoubleToken() { + byte[] bytes = input; + int offset = position; + int inputLength = bytes.length; + if (offset >= inputLength) { + return readDoubleFallback(offset); + } + int ch = bytes[offset]; + if (ch == '-') { + return readSignedDoubleToken(offset); + } + return readPositiveDoubleToken(bytes, offset, inputLength, ch); + } + + private double readPositiveDoubleToken(byte[] bytes, int offset, int inputLength, int ch) { + int start = offset; + long unscaled = 0; + int scale = 0; + if (ch == '0') { + offset++; + if (offset < inputLength) { + ch = bytes[offset]; + if (ch >= '0' && ch <= '9') { + return readDoubleFallback(start); + } + } + } else if (ch >= '1' && ch <= '9') { + unscaled = ch - '0'; + offset++; + while (offset + 1 < inputLength) { + int high = bytes[offset] - '0'; + int low = bytes[offset + 1] - '0'; + if (high < 0 || high > 9 || low < 0 || low > 9) { + break; + } + int pair = high * 10 + low; + if (unscaled > LONG_MAX_DIV_100 + || (unscaled == LONG_MAX_DIV_100 && pair > LONG_MAX_MOD_100)) { + return readDoubleFallback(start); + } + unscaled = unscaled * 100 + pair; + offset += 2; + } + if (offset < inputLength) { + int digit = bytes[offset] - '0'; + if (digit >= 0 && digit <= 9) { + if (unscaled > LONG_MAX_DIV_10 + || (unscaled == LONG_MAX_DIV_10 && digit > LONG_MAX_MOD_10)) { + return readDoubleFallback(start); + } + unscaled = unscaled * 10 + digit; + offset++; + } + } + } else { + return readDoubleFallback(start); + } + if (offset < inputLength && bytes[offset] == '.') { + offset++; + int fractionStart = offset; + while (offset + 1 < inputLength) { + int high = bytes[offset] - '0'; + int low = bytes[offset + 1] - '0'; + if (high < 0 || high > 9 || low < 0 || low > 9) { + break; + } + int pair = high * 10 + low; + if (unscaled > LONG_MAX_DIV_100 + || (unscaled == LONG_MAX_DIV_100 && pair > LONG_MAX_MOD_100)) { + return readDoubleFallback(start); + } + unscaled = unscaled * 100 + pair; + scale += 2; + offset += 2; + } + if (offset < inputLength) { + int digit = bytes[offset] - '0'; + if (digit >= 0 && digit <= 9) { + if (unscaled > LONG_MAX_DIV_10 + || (unscaled == LONG_MAX_DIV_10 && digit > LONG_MAX_MOD_10)) { + return readDoubleFallback(start); + } + unscaled = unscaled * 10 + digit; + scale++; + offset++; + } + } + if (offset == fractionStart) { + return readDoubleFallback(start); + } + } + return finishDoubleToken(bytes, offset, inputLength, start, unscaled, scale); + } + + private double readSignedDoubleToken(int start) { + byte[] bytes = input; + int offset = start + 1; + int inputLength = bytes.length; + if (offset >= inputLength) { + return readDoubleFallback(start); + } + int ch = bytes[offset]; + long unscaled = 0; + int scale = 0; + if (ch == '0') { + offset++; + if (offset < inputLength) { + ch = bytes[offset]; + if (ch >= '0' && ch <= '9') { + return readDoubleFallback(start); + } + } + } else if (ch >= '1' && ch <= '9') { + unscaled = ch - '0'; + offset++; + while (offset + 1 < inputLength) { + int high = bytes[offset] - '0'; + int low = bytes[offset + 1] - '0'; + if (high < 0 || high > 9 || low < 0 || low > 9) { + break; + } + int pair = high * 10 + low; + if (unscaled > LONG_MAX_DIV_100 + || (unscaled == LONG_MAX_DIV_100 && pair > LONG_MAX_MOD_100)) { + return readDoubleFallback(start); + } + unscaled = unscaled * 100 + pair; + offset += 2; + } + if (offset < inputLength) { + int digit = bytes[offset] - '0'; + if (digit >= 0 && digit <= 9) { + if (unscaled > LONG_MAX_DIV_10 + || (unscaled == LONG_MAX_DIV_10 && digit > LONG_MAX_MOD_10)) { + return readDoubleFallback(start); + } + unscaled = unscaled * 10 + digit; + offset++; + } + } + } else { + return readDoubleFallback(start); + } + if (offset < inputLength && bytes[offset] == '.') { + offset++; + int fractionStart = offset; + while (offset + 1 < inputLength) { + int high = bytes[offset] - '0'; + int low = bytes[offset + 1] - '0'; + if (high < 0 || high > 9 || low < 0 || low > 9) { + break; + } + int pair = high * 10 + low; + if (unscaled > LONG_MAX_DIV_100 + || (unscaled == LONG_MAX_DIV_100 && pair > LONG_MAX_MOD_100)) { + return readDoubleFallback(start); + } + unscaled = unscaled * 100 + pair; + scale += 2; + offset += 2; + } + if (offset < inputLength) { + int digit = bytes[offset] - '0'; + if (digit >= 0 && digit <= 9) { + if (unscaled > LONG_MAX_DIV_10 + || (unscaled == LONG_MAX_DIV_10 && digit > LONG_MAX_MOD_10)) { + return readDoubleFallback(start); + } + unscaled = unscaled * 10 + digit; + scale++; + offset++; + } + } + if (offset == fractionStart) { + return readDoubleFallback(start); + } + } + return finishSignedDoubleToken(bytes, offset, inputLength, start, unscaled, scale); + } + + private double finishDoubleToken( + byte[] bytes, int offset, int inputLength, int start, long unscaled, int scale) { + if (offset < inputLength) { + int ch = bytes[offset]; + if (ch == 'e' || ch == 'E') { + return readDoubleFallback(start); + } + } + position = offset; + return BigDecimal.valueOf(unscaled, scale).doubleValue(); + } + + private double finishSignedDoubleToken( + byte[] bytes, int offset, int inputLength, int start, long unscaled, int scale) { + if (offset < inputLength) { + int ch = bytes[offset]; + if (ch == 'e' || ch == 'E') { + return readDoubleFallback(start); + } + } + position = offset; + if (unscaled == 0) { + return -0.0d; + } + return BigDecimal.valueOf(-unscaled, scale).doubleValue(); + } + + private double readDoubleFallback(int start) { + position = start; + return Double.parseDouble(readNumber()); + } + @Override public int readFieldNameInt() { skipWhitespaceFast(); @@ -610,7 +1122,7 @@ public String readNextNullableString() { if (ch == 'n' && tryReadNullLiteral()) { return null; } - if (!isWhitespace(ch)) { + if (ch > ' ' || !isWhitespace(ch)) { return readStringToken(); } } @@ -630,6 +1142,28 @@ public String readNullableStringToken() { return readStringToken(); } + public LocalDate readIsoLocalDate() { + skipWhitespaceFast(); + int mark = position; + try { + return readIsoLocalDateToken(); + } catch (RuntimeException e) { + position = mark; + throw e; + } + } + + public OffsetDateTime readIsoOffsetDateTime() { + skipWhitespaceFast(); + int mark = position; + try { + return readIsoOffsetDateTimeToken(); + } catch (RuntimeException e) { + position = mark; + throw e; + } + } + private String readStringToken() { byte[] bytes = input; int inputLength = bytes.length; @@ -638,28 +1172,58 @@ private String readStringToken() { } int start = position; int offset = start; + int doubleWordEnd = inputLength - (Long.BYTES << 1); + while (offset <= doubleWordEnd) { + long stopMask = asciiStringStopMask(LittleEndian.getInt64(bytes, offset)); + if (stopMask != 0) { + int stop = offset + (Long.numberOfTrailingZeros(stopMask) >>> 3); + int ch = bytes[stop]; + if (ch == '"') { + position = stop + 1; + return newLatin1String(start, stop); + } + return readStringStop(start, stop, ch); + } + int nextOffset = offset + Long.BYTES; + stopMask = asciiStringStopMask(LittleEndian.getInt64(bytes, nextOffset)); + if (stopMask != 0) { + int stop = nextOffset + (Long.numberOfTrailingZeros(stopMask) >>> 3); + int ch = bytes[stop]; + if (ch == '"') { + position = stop + 1; + return newLatin1String(start, stop); + } + return readStringStop(start, stop, ch); + } + offset = nextOffset + Long.BYTES; + } int wordEnd = inputLength - Long.BYTES; while (offset <= wordEnd) { - long stopMask = stringStopMask(LittleEndian.getInt64(bytes, offset)); + long stopMask = asciiStringStopMask(LittleEndian.getInt64(bytes, offset)); if (stopMask == 0) { offset += Long.BYTES; continue; } int stop = offset + (Long.numberOfTrailingZeros(stopMask) >>> 3); - int ch = bytes[stop] & 0xFF; + int ch = bytes[stop]; if (ch == '"') { position = stop + 1; return newLatin1String(start, stop); } return readStringStop(start, stop, ch); } + return readStringTokenTail(start, offset, inputLength); + } + + private String readStringTokenTail(int start, int offset, int inputLength) { + byte[] bytes = input; if (offset + Integer.BYTES <= inputLength) { int stopMask = stringStopMask(LittleEndian.getInt32(bytes, offset)); if (stopMask == 0) { offset += Integer.BYTES; } else { int stop = offset + (Integer.numberOfTrailingZeros(stopMask) >>> 3); - int ch = bytes[stop] & 0xFF; + int ch = bytes[stop]; if (ch == '"') { position = stop + 1; return newLatin1String(start, stop); @@ -684,19 +1248,210 @@ private String readStringToken() { throw error("Unterminated string"); } + private LocalDate readIsoLocalDateToken() { + byte[] bytes = input; + int offset = position; + int length = bytes.length; + if (offset + 12 > length || bytes[offset++] != '"') { + throw error("Expected string"); + } + int dateStart = offset; + if (bytes[dateStart + 4] != '-' || bytes[dateStart + 7] != '-') { + throw new IllegalArgumentException(); + } + int year = parse4(bytes, dateStart); + int month = parse2(bytes, dateStart + 5); + int day = parse2(bytes, dateStart + 8); + int end = dateStart + 10; + int ch = bytes[end]; + if (ch == '"') { + position = end + 1; + return LocalDate.of(year, month, day); + } + if (ch == 'T') { + position = scanSimpleStringTail(bytes, end + 1); + return LocalDate.of(year, month, day); + } + throw new IllegalArgumentException(); + } + + private OffsetDateTime readIsoOffsetDateTimeToken() { + byte[] bytes = input; + int offset = position; + int length = bytes.length; + if (offset + 19 > length || bytes[offset++] != '"') { + throw error("Expected string"); + } + int start = offset; + if (bytes[start + 4] != '-' + || bytes[start + 7] != '-' + || bytes[start + 10] != 'T' + || bytes[start + 13] != ':') { + throw new IllegalArgumentException(); + } + int year = parse4(bytes, start); + int month = parse2(bytes, start + 5); + int day = parse2(bytes, start + 8); + int hour = parse2(bytes, start + 11); + int minute = parse2(bytes, start + 14); + return readIsoOffsetDateTimeTail(bytes, start + 16, length, year, month, day, hour, minute); + } + + private OffsetDateTime readIsoOffsetDateTimeTail( + byte[] bytes, int index, int length, int year, int month, int day, int hour, int minute) { + int second = 0; + int nano = 0; + if (index < length && bytes[index] == ':') { + second = parse2(bytes, index + 1); + index += 3; + if (index < length && bytes[index] == '.') { + int fractionStart = index + 1; + int fractionEnd = fractionStart; + while (fractionEnd < length && isDigit(bytes[fractionEnd])) { + fractionEnd++; + } + if (fractionEnd == fractionStart || fractionEnd - fractionStart > 9) { + throw new IllegalArgumentException(); + } + nano = parseNano(bytes, fractionStart, fractionEnd); + index = fractionEnd; + } + } + if (index < length && bytes[index] == 'Z') { + if (index + 1 >= length || bytes[index + 1] != '"') { + throw new IllegalArgumentException(); + } + position = index + 2; + return OffsetDateTime.of(year, month, day, hour, minute, second, nano, ZoneOffset.UTC); + } + return readIsoOffsetDateTimeOffsetTail( + bytes, index, length, year, month, day, hour, minute, second, nano); + } + + private OffsetDateTime readIsoOffsetDateTimeOffsetTail( + byte[] bytes, + int index, + int length, + int year, + int month, + int day, + int hour, + int minute, + int second, + int nano) { + long offsetAndEnd = parseOffsetAndEnd(bytes, index, length); + position = (int) offsetAndEnd; + return OffsetDateTime.of( + year, + month, + day, + hour, + minute, + second, + nano, + ZoneOffset.ofTotalSeconds((int) (offsetAndEnd >> 32))); + } + + private int scanSimpleStringTail(byte[] bytes, int offset) { + int length = bytes.length; + while (offset < length) { + int b = bytes[offset++]; + if (b == '"') { + return offset; + } + if (b == '\\' || b < 0x20 || b < 0) { + throw new IllegalArgumentException(); + } + } + throw error("Unterminated string"); + } + + private static long parseOffsetAndEnd(byte[] bytes, int index, int length) { + int offset = bytes[index]; + if (offset == 'Z') { + if (index + 1 >= length || bytes[index + 1] != '"') { + throw new IllegalArgumentException(); + } + return ((long) (index + 2)) & 0xFFFF_FFFFL; + } + if (offset != '+' && offset != '-') { + throw new IllegalArgumentException(); + } + if (index + 6 >= length || bytes[index + 3] != ':') { + throw new IllegalArgumentException(); + } + int hour = parse2(bytes, index + 1); + int minute = parse2(bytes, index + 4); + int second = 0; + int end = index + 6; + if (bytes[end] == ':') { + if (end + 3 >= length) { + throw new IllegalArgumentException(); + } + second = parse2(bytes, end + 1); + end += 3; + } + if (bytes[end] != '"') { + throw new IllegalArgumentException(); + } + int total = hour * 3600 + minute * 60 + second; + if (offset == '-') { + total = -total; + } + return ((long) total << 32) | ((long) (end + 1) & 0xFFFF_FFFFL); + } + + private static int parseNano(byte[] bytes, int start, int end) { + int nano = 0; + for (int i = start; i < end; i++) { + nano = nano * 10 + bytes[i] - '0'; + } + for (int i = end - start; i < 9; i++) { + nano *= 10; + } + return nano; + } + + private static int parse4(byte[] bytes, int index) { + return parse2(bytes, index) * 100 + parse2(bytes, index + 2); + } + + private static int parse2(byte[] bytes, int index) { + int high = bytes[index] - '0'; + int low = bytes[index + 1] - '0'; + if (high < 0 || high > 9 || low < 0 || low > 9) { + throw new IllegalArgumentException(); + } + return high * 10 + low; + } + + private static boolean isDigit(byte b) { + return b >= '0' && b <= '9'; + } + private String readStringStop(int start, int stop, int ch) { + if (ch < 0) { + return readStringTokenTail(start, stop, input.length); + } position = stop + 1; if (ch == '\\') { - StringBuilder builder = newStringBuilder(start, stop); - appendLatin1(builder, start, stop); - appendEscape(builder); - return readStringTail(builder); + int out = stop - start; + byte[] bytes = stringDecodeBuffer; + if (bytes.length < out) { + bytes = growStringDecodeBuffer(bytes, out); + } + System.arraycopy(input, start, bytes, 0, out); + return readStringLatin1Tail(bytes, out, ch); } throw error("Control character in string"); } - private StringBuilder newStringBuilder(int start, int stop) { - return new StringBuilder(Math.max(16, stop - start + 16)); + private static long asciiStringStopMask(long word) { + // ASCII segments can use the cheaper UTF-8-style syntax scan. High-bit bytes stop this fast + // path and continue through the precise Latin1 scanner, where non-ASCII Latin1 remains valid + // and raw control bytes still fail. + long syntaxStop = ((word ^ QUOTE_BYTES) - BYTE_ONES) | ((word ^ BACKSLASH_BYTES) - BYTE_ONES); + return (syntaxStop | word | (word - CONTROL_LIMIT_BYTES)) & BYTE_HIGH_BITS; } private static long stringStopMask(long word) { @@ -854,6 +1609,37 @@ private boolean tryReadNextRawToken3(long prefix, long prefixMask, int suffix, i return false; } + public boolean tryReadNextFieldNameToken8( + long prefix, long suffix, long suffixMask, int tokenLength) { + return tryReadNextRawToken8(prefix, suffix, suffixMask, tokenLength); + } + + private boolean tryReadNextRawToken8(long prefix, long suffix, long suffixMask, int tokenLength) { + byte[] bytes = input; + int mark = position; + int suffixOffset = mark + Long.BYTES; + if (mark + tokenLength <= bytes.length + && LittleEndian.getInt64(bytes, mark) == prefix + && readTokenSuffix(bytes, suffixOffset, tokenLength, suffixMask) == suffix) { + position = mark + tokenLength; + return true; + } + return false; + } + + private static long readTokenSuffix( + byte[] bytes, int suffixOffset, int tokenLength, long suffixMask) { + if (suffixOffset + Long.BYTES <= bytes.length) { + return LittleEndian.getInt64(bytes, suffixOffset) & suffixMask; + } + int suffixLength = tokenLength - Long.BYTES; + long suffix = 0; + for (int i = 0; i < suffixLength; i++) { + suffix |= (long) (bytes[suffixOffset + i] & 0xFF) << (i << 3); + } + return suffix; + } + private boolean tryReadFieldNameColonAt( int mark, long expectedHash, long expectedMask, int expectedLength) { byte[] bytes = input; @@ -1044,28 +1830,6 @@ private String newLatin1String(int start, int end) { return StringSerializer.newLatin1StringZeroCopy(bytes); } - private String readStringTail(StringBuilder builder) { - while (position < input.length) { - int ch = input[position++] & 0xFF; - if (ch == '"') { - return builder.toString(); - } else if (ch == '\\') { - appendEscape(builder); - } else if (ch < 0x20) { - throw error("Control character in string"); - } else { - builder.append((char) ch); - } - } - throw error("Unterminated string"); - } - - private void appendLatin1(StringBuilder builder, int start, int end) { - for (int i = start; i < end; i++) { - builder.append((char) (input[i] & 0xFF)); - } - } - private void skipWhitespaceFast() { while (position < input.length) { int ch = input[position]; @@ -1094,6 +1858,159 @@ private boolean startsWithAscii(String value) { return true; } + private String readStringLatin1Tail(byte[] bytes, int out, int ch) { + while (true) { + if (ch == '"') { + return finishDecodedString(bytes, out, false); + } + if (ch == '\\') { + char escaped = readEscapedStringChar(); + if (Character.isHighSurrogate(escaped)) { + char low = readLowSurrogateEscape(); + bytes = widenStringDecodeBuffer(bytes, out); + out <<= 1; + bytes = ensureStringDecodeCapacity(bytes, out + 4); + out = putUtf16Char(bytes, out, escaped); + out = putUtf16Char(bytes, out, low); + return readStringUtf16Tail(bytes, out); + } + if (Character.isLowSurrogate(escaped)) { + throw error("Unpaired low surrogate escape"); + } + if (escaped <= 0xFF) { + bytes = ensureStringDecodeCapacity(bytes, out + 1); + bytes[out++] = (byte) escaped; + } else { + bytes = widenStringDecodeBuffer(bytes, out); + out <<= 1; + bytes = ensureStringDecodeCapacity(bytes, out + 2); + out = putUtf16Char(bytes, out, escaped); + return readStringUtf16Tail(bytes, out); + } + } else if (ch < 0x20) { + throw error("Control character in string"); + } else { + bytes = ensureStringDecodeCapacity(bytes, out + 1); + bytes[out++] = (byte) ch; + } + if (position >= input.length) { + throw error("Unterminated string"); + } + ch = input[position++] & 0xFF; + } + } + + private String readStringUtf16Tail(byte[] bytes, int out) { + while (position < input.length) { + int ch = input[position++] & 0xFF; + if (ch == '"') { + return finishDecodedString(bytes, out, true); + } + if (ch == '\\') { + char escaped = readEscapedStringChar(); + if (Character.isHighSurrogate(escaped)) { + char low = readLowSurrogateEscape(); + bytes = ensureStringDecodeCapacity(bytes, out + 4); + out = putUtf16Char(bytes, out, escaped); + out = putUtf16Char(bytes, out, low); + } else if (Character.isLowSurrogate(escaped)) { + throw error("Unpaired low surrogate escape"); + } else { + bytes = ensureStringDecodeCapacity(bytes, out + 2); + out = putUtf16Char(bytes, out, escaped); + } + } else if (ch < 0x20) { + throw error("Control character in string"); + } else { + bytes = ensureStringDecodeCapacity(bytes, out + 2); + out = putUtf16Char(bytes, out, (char) ch); + } + } + throw error("Unterminated string"); + } + + private char readEscapedStringChar() { + if (position >= input.length) { + throw error("Unterminated escape"); + } + char escaped = (char) (input[position++] & 0xFF); + switch (escaped) { + case '"': + case '\\': + case '/': + return escaped; + case 'b': + return '\b'; + case 'f': + return '\f'; + case 'n': + return '\n'; + case 'r': + return '\r'; + case 't': + return '\t'; + case 'u': + return readUnicodeEscape(); + default: + throw error("Invalid escape"); + } + } + + private char readLowSurrogateEscape() { + if (position + 2 > input.length || input[position] != '\\' || input[position + 1] != 'u') { + throw error("Unpaired high surrogate escape"); + } + position += 2; + char low = readUnicodeEscape(); + if (!Character.isLowSurrogate(low)) { + throw error("Unpaired high surrogate escape"); + } + return low; + } + + private String finishDecodedString(byte[] bytes, int length, boolean utf16) { + // The decode buffer is reader-owned reusable storage; returned Strings must own exact bytes. + byte[] result = new byte[length]; + System.arraycopy(bytes, 0, result, 0, length); + return utf16 + ? StringSerializer.newUtf16StringZeroCopy(result) + : StringSerializer.newLatin1StringZeroCopy(result); + } + + private byte[] ensureStringDecodeCapacity(byte[] bytes, int capacity) { + if (bytes.length < capacity) { + return growStringDecodeBuffer(bytes, capacity); + } + return bytes; + } + + private byte[] growStringDecodeBuffer(byte[] bytes, int capacity) { + int newCapacity = Math.max(capacity, bytes.length << 1); + byte[] grown = Arrays.copyOf(bytes, newCapacity); + stringDecodeBuffer = grown; + return grown; + } + + private byte[] widenStringDecodeBuffer(byte[] bytes, int length) { + int utf16Length = length << 1; + bytes = ensureStringDecodeCapacity(bytes, utf16Length); + for (int i = length - 1, pos = utf16Length - 2; i >= 0; i--, pos -= 2) { + putUtf16Char(bytes, pos, (char) (bytes[i] & 0xFF)); + } + return bytes; + } + + private static int putUtf16Char(byte[] bytes, int pos, char value) { + if (LITTLE_ENDIAN) { + bytes[pos] = (byte) value; + bytes[pos + 1] = (byte) (value >>> 8); + } else { + bytes[pos] = (byte) (value >>> 8); + bytes[pos + 1] = (byte) value; + } + return pos + 2; + } + private void rejectLeadingDigitFast() { if (position < input.length) { int ch = input[position]; diff --git a/java/fory-json/src/main/java/org/apache/fory/json/reader/Utf16JsonReader.java b/java/fory-json/src/main/java/org/apache/fory/json/reader/Utf16JsonReader.java index b2f0f6a058..6cd1abdd6c 100644 --- a/java/fory-json/src/main/java/org/apache/fory/json/reader/Utf16JsonReader.java +++ b/java/fory-json/src/main/java/org/apache/fory/json/reader/Utf16JsonReader.java @@ -19,15 +19,40 @@ package org.apache.fory.json.reader; +import java.time.LocalDate; +import java.time.OffsetDateTime; +import java.time.ZoneOffset; +import java.util.Arrays; import org.apache.fory.json.meta.JsonFieldInfo; import org.apache.fory.json.meta.JsonFieldNameHash; import org.apache.fory.json.meta.JsonFieldTable; +import org.apache.fory.memory.LittleEndian; +import org.apache.fory.memory.NativeByteOrder; import org.apache.fory.serializer.StringSerializer; public final class Utf16JsonReader extends JsonReader { + private static final int INITIAL_STRING_DECODE_BUFFER_SIZE = 1024; + private static final int RETAINED_STRING_DECODE_BUFFER_SIZE = 8192; + private static final boolean LITTLE_ENDIAN = NativeByteOrder.IS_LITTLE_ENDIAN; + private static final long BYTE_ONES = 0x0101010101010101L; + private static final long BYTE_HIGH_BITS = 0x8080808080808080L; + private static final long BACKSLASH_BYTES = 0x5c5c5c5c5c5c5c5cL; + private static final long CONTROL_LIMIT_BYTES = 0x2020202020202020L; + private static final long QUOTE_BYTES = 0x2222222222222222L; + private static final long UTF16_PAIR_MASK = 0x0000FFFF0000FFFFL; + private static final long UTF16_BYTE_MASK = 0x00FF00FF00FF00FFL; + private static final long UTF16_ONES = 0x0001000100010001L; + private static final long UTF16_HIGH_BITS = 0x8000800080008000L; + private static final long UTF16_QUOTE_CHARS = 0x0022002200220022L; + private static final long UTF16_BACKSLASH_CHARS = 0x005c005c005c005cL; + private static final long UTF16_CONTROL_LIMIT = 0x0020002000200020L; + private static final long UTF16_NON_LATIN_BYTES = 0xFF00FF00FF00FF00L; + private static final long UTF16_SURROGATE_MASK = 0xf800f800f800f800L; + private static final long UTF16_SURROGATE_PREFIX = 0xd800d800d800d800L; private String input; private byte[] bytes; private int length; + private byte[] stringDecodeBuffer = new byte[INITIAL_STRING_DECODE_BUFFER_SIZE]; public Utf16JsonReader() { input = ""; @@ -56,16 +81,34 @@ public Utf16JsonReader reset(String input) { return this; } + public Utf16JsonReader resetUtf16Bytes(String input, byte[] bytes) { + int length = input.length(); + if (length > (Integer.MAX_VALUE >>> 1)) { + throw new IllegalArgumentException("String is too large"); + } + if (bytes.length < (length << 1)) { + throw new IllegalArgumentException("UTF16 byte array is too small"); + } + this.input = input; + this.bytes = bytes; + this.length = length; + position = 0; + return this; + } + public void clear() { input = ""; bytes = null; length = 0; position = 0; + if (stringDecodeBuffer.length > RETAINED_STRING_DECODE_BUFFER_SIZE) { + stringDecodeBuffer = new byte[RETAINED_STRING_DECODE_BUFFER_SIZE]; + } } public boolean consumeToken(char expected) { skipWhitespaceFast(); - if (position < length && charAtFast(position) == expected) { + if (position < length && asciiAtFast(position) == expected) { position++; return true; } @@ -73,7 +116,7 @@ public boolean consumeToken(char expected) { } public boolean consumeNextToken(char expected) { - if (position < length && charAtFast(position) == expected) { + if (position < length && asciiAtFast(position) == expected) { position++; return true; } @@ -87,7 +130,7 @@ public void expectToken(char expected) { } public void expectNextToken(char expected) { - if (position < length && charAtFast(position) == expected) { + if (position < length && asciiAtFast(position) == expected) { position++; return; } @@ -101,7 +144,7 @@ private void expectNextTokenSlow(char expected) { public boolean consumeNextCommaOrEndObject() { int inputLength = length; if (position < inputLength) { - char ch = charAtFast(position); + int ch = asciiAtFast(position); if (ch == ',') { position++; return true; @@ -120,7 +163,7 @@ public boolean consumeNextCommaOrEndObject() { private boolean consumeNextCommaOrEndObjectSlow(int inputLength) { skipWhitespaceFast(); if (position < inputLength) { - char ch = charAtFast(position); + int ch = asciiAtFast(position); if (ch == ',') { position++; return true; @@ -136,7 +179,7 @@ private boolean consumeNextCommaOrEndObjectSlow(int inputLength) { public boolean consumeNextCommaOrEndArray() { int inputLength = length; if (position < inputLength) { - char ch = charAtFast(position); + int ch = asciiAtFast(position); if (ch == ',') { position++; return true; @@ -155,7 +198,7 @@ public boolean consumeNextCommaOrEndArray() { private boolean consumeNextCommaOrEndArraySlow(int inputLength) { skipWhitespaceFast(); if (position < inputLength) { - char ch = charAtFast(position); + int ch = asciiAtFast(position); if (ch == ',') { position++; return true; @@ -175,7 +218,7 @@ public boolean tryReadNullToken() { public boolean tryReadNextNullToken() { if (position < length) { - char ch = charAtFast(position); + int ch = asciiAtFast(position); if (ch == 'n') { return tryReadNullLiteral(); } @@ -200,12 +243,16 @@ public boolean readBooleanValue() { } public boolean readNextBooleanValue() { - if (position < length && !isWhitespace(charAtFast(position))) { + if (position < length && !isWhitespace(asciiAtFast(position))) { return readBooleanToken(); } return readBooleanValue(); } + public boolean readBooleanTokenValue() { + return readBooleanToken(); + } + private boolean readBooleanToken() { if (startsWithAscii("true")) { position += 4; @@ -223,12 +270,16 @@ public int readIntValue() { } public int readNextIntValue() { - if (position < length && !isWhitespace(charAtFast(position))) { + if (position < length && !isWhitespace(asciiAtFast(position))) { return readIntToken(); } return readIntValue(); } + public int readIntTokenValue() { + return readIntToken(); + } + private int readIntToken() { int offset = position; int inputLength = length; @@ -336,12 +387,16 @@ public long readLongValue() { } public long readNextLongValue() { - if (position < length && !isWhitespace(charAtFast(position))) { + if (position < length && !isWhitespace(asciiAtFast(position))) { return readLongToken(); } return readLongValue(); } + public long readLongTokenValue() { + return readLongToken(); + } + private long readLongToken() { int offset = position; int inputLength = length; @@ -586,9 +641,10 @@ public String readNullableString() { public String readNextNullableString() { if (position < length) { - char ch = charAtFast(position); + int ch = asciiAtFast(position); if (ch == '"') { - return readStringToken(); + position++; + return readStringAfterQuote(); } if (ch == 'n' && tryReadNullLiteral()) { return null; @@ -600,27 +656,409 @@ public String readNextNullableString() { return readNullableString(); } + public String readNullableStringToken() { + if (position < length) { + int ch = asciiAtFast(position); + if (ch == '"') { + position++; + return readStringAfterQuote(); + } + if (ch == 'n' && tryReadNullLiteral()) { + return null; + } + } + return readStringToken(); + } + + public LocalDate readIsoLocalDate() { + skipWhitespaceFast(); + int mark = position; + try { + return readIsoLocalDateToken(); + } catch (RuntimeException e) { + position = mark; + throw e; + } + } + + public OffsetDateTime readIsoOffsetDateTime() { + skipWhitespaceFast(); + int mark = position; + try { + return readIsoOffsetDateTimeToken(); + } catch (RuntimeException e) { + position = mark; + throw e; + } + } + private String readStringToken() { if (position >= length || charAtFast(position++) != '"') { throw error("Expected string"); } + return readStringAfterQuote(); + } + + private String readStringAfterQuote() { int start = position; - StringBuilder builder = null; + if (LITTLE_ENDIAN && bytes != null) { + return readUtf16StringToken(start); + } + return readStringLoop(start, false); + } + + private String readUtf16StringToken(int start) { + byte[] localBytes = bytes; + int inputLength = length; + int offset = start; + boolean nonLatin = false; + int doubleWordEnd = inputLength - 8; + while (offset <= doubleWordEnd) { + long word = LittleEndian.getInt64(localBytes, offset << 1); + long nonLatinBytes = word & UTF16_NON_LATIN_BYTES; + long stopMask = utf16StringStopMask(word, nonLatinBytes); + if (stopMask != 0) { + int stop = offset + (Long.numberOfTrailingZeros(stopMask) >>> 4); + boolean currentNonLatin = + nonLatin || nonLatinBytes != 0 && hasNonLatinBeforeStop(word, stop - offset); + char ch = charAtFast(stop); + if (Character.isHighSurrogate(ch)) { + currentNonLatin = true; + } + String value = readUtf16StringStop(localBytes, start, stop, currentNonLatin, ch); + if (value != null) { + return value; + } + nonLatin = currentNonLatin; + offset = position; + continue; + } + nonLatin |= nonLatinBytes != 0; + int nextOffset = offset + 4; + word = LittleEndian.getInt64(localBytes, nextOffset << 1); + nonLatinBytes = word & UTF16_NON_LATIN_BYTES; + stopMask = utf16StringStopMask(word, nonLatinBytes); + if (stopMask != 0) { + int stop = nextOffset + (Long.numberOfTrailingZeros(stopMask) >>> 4); + boolean currentNonLatin = + nonLatin || nonLatinBytes != 0 && hasNonLatinBeforeStop(word, stop - nextOffset); + char ch = charAtFast(stop); + if (Character.isHighSurrogate(ch)) { + currentNonLatin = true; + } + String value = readUtf16StringStop(localBytes, start, stop, currentNonLatin, ch); + if (value != null) { + return value; + } + nonLatin = currentNonLatin; + offset = position; + continue; + } + nonLatin |= nonLatinBytes != 0; + offset = nextOffset + 4; + } + int wordEnd = inputLength - 4; + while (offset <= wordEnd) { + long word = LittleEndian.getInt64(localBytes, offset << 1); + long nonLatinBytes = word & UTF16_NON_LATIN_BYTES; + long stopMask = utf16StringStopMask(word, nonLatinBytes); + if (stopMask == 0) { + nonLatin |= nonLatinBytes != 0; + offset += 4; + continue; + } + int stop = offset + (Long.numberOfTrailingZeros(stopMask) >>> 4); + boolean currentNonLatin = + nonLatin || nonLatinBytes != 0 && hasNonLatinBeforeStop(word, stop - offset); + char ch = charAtFast(stop); + if (Character.isHighSurrogate(ch)) { + currentNonLatin = true; + } + String value = readUtf16StringStop(localBytes, start, stop, currentNonLatin, ch); + if (value != null) { + return value; + } + nonLatin = currentNonLatin; + offset = position; + } + position = offset; + return readStringLoop(start, nonLatin); + } + + private String readUtf16StringStop( + byte[] localBytes, int start, int stop, boolean nonLatin, char ch) { + if (ch == '"') { + position = stop + 1; + // Preserve Java compact-string invariants: a LATIN1-representable token must keep the JDK's + // normal LATIN1 representation, otherwise equality/hash behavior can differ for map/set keys. + return nonLatin ? newUtf16String(localBytes, start, stop) : input.substring(start, stop); + } + if (ch == '\\') { + position = stop + 1; + return readEscapedStringTail(start, stop, nonLatin); + } + if (ch < 0x20) { + position = stop; + throw error("Control character in string"); + } + if (Character.isHighSurrogate(ch)) { + int lowOffset = stop + 1; + if (lowOffset >= length || !Character.isLowSurrogate(charAtFast(lowOffset))) { + position = stop; + throw error("Unpaired high surrogate in string"); + } + position = lowOffset + 1; + return null; + } + position = stop; + throw error("Unpaired low surrogate in string"); + } + + private static boolean hasNonLatinBeforeStop(long word, int chars) { + long mask = chars == 4 ? -1L : (1L << (chars << 4)) - 1; + return (word & mask & UTF16_NON_LATIN_BYTES) != 0; + } + + private static String newUtf16String(byte[] localBytes, int start, int stop) { + byte[] valueBytes = new byte[(stop - start) << 1]; + System.arraycopy(localBytes, start << 1, valueBytes, 0, valueBytes.length); + return StringSerializer.newUtf16StringZeroCopy(valueBytes); + } + + private String readEscapedStringTail(int start, int stop, boolean nonLatin) { + if (nonLatin) { + int out = (stop - start) << 1; + byte[] outBytes = stringDecodeBuffer; + if (outBytes.length < out) { + outBytes = growStringDecodeBuffer(outBytes, out); + } + copyPrefixUtf16(start, stop, outBytes); + return readStringUtf16Tail(outBytes, out, '\\'); + } + int out = stop - start; + byte[] outBytes = stringDecodeBuffer; + if (outBytes.length < out) { + outBytes = growStringDecodeBuffer(outBytes, out); + } + copyPrefixLatin1(start, stop, outBytes); + return readStringLatin1Tail(outBytes, out, '\\'); + } + + private void copyPrefixLatin1(int start, int stop, byte[] outBytes) { + for (int i = start, out = 0; i < stop; i++) { + outBytes[out++] = (byte) charAtFast(i); + } + } + + private void copyPrefixUtf16(int start, int stop, byte[] outBytes) { + byte[] localBytes = bytes; + int length = (stop - start) << 1; + if (localBytes != null) { + System.arraycopy(localBytes, start << 1, outBytes, 0, length); + return; + } + for (int i = start, out = 0; i < stop; i++) { + out = putUtf16Char(outBytes, out, charAtFast(i)); + } + } + + private String readStringLatin1Tail(byte[] outBytes, int out, int ch) { + while (true) { + if (ch == '"') { + return finishDecodedString(outBytes, out, false); + } + if (ch == '\\') { + char escaped = readEscapedStringChar(); + if (Character.isHighSurrogate(escaped)) { + char low = readLowSurrogateEscape(); + outBytes = widenStringDecodeBuffer(outBytes, out); + out <<= 1; + outBytes = ensureStringDecodeCapacity(outBytes, out + 4); + out = putUtf16Char(outBytes, out, escaped); + out = putUtf16Char(outBytes, out, low); + return readStringUtf16Tail(outBytes, out, nextStringChar()); + } + if (Character.isLowSurrogate(escaped)) { + throw error("Unpaired low surrogate escape"); + } + if (escaped <= 0xFF) { + outBytes = ensureStringDecodeCapacity(outBytes, out + 1); + outBytes[out++] = (byte) escaped; + } else { + outBytes = widenStringDecodeBuffer(outBytes, out); + out <<= 1; + outBytes = ensureStringDecodeCapacity(outBytes, out + 2); + out = putUtf16Char(outBytes, out, escaped); + return readStringUtf16Tail(outBytes, out, nextStringChar()); + } + } else if (ch < 0x20) { + throw error("Control character in string"); + } else if (Character.isHighSurrogate((char) ch)) { + char low = readRawLowSurrogate(); + outBytes = widenStringDecodeBuffer(outBytes, out); + out <<= 1; + outBytes = ensureStringDecodeCapacity(outBytes, out + 4); + out = putUtf16Char(outBytes, out, (char) ch); + out = putUtf16Char(outBytes, out, low); + return readStringUtf16Tail(outBytes, out, nextStringChar()); + } else if (Character.isLowSurrogate((char) ch)) { + throw error("Unpaired low surrogate in string"); + } else if (ch <= 0xFF) { + outBytes = ensureStringDecodeCapacity(outBytes, out + 1); + outBytes[out++] = (byte) ch; + } else { + outBytes = widenStringDecodeBuffer(outBytes, out); + out <<= 1; + outBytes = ensureStringDecodeCapacity(outBytes, out + 2); + out = putUtf16Char(outBytes, out, (char) ch); + return readStringUtf16Tail(outBytes, out, nextStringChar()); + } + ch = nextStringChar(); + } + } + + private String readStringUtf16Tail(byte[] outBytes, int out, int ch) { + while (true) { + if (ch == '"') { + return finishDecodedString(outBytes, out, true); + } + if (ch == '\\') { + char escaped = readEscapedStringChar(); + if (Character.isHighSurrogate(escaped)) { + char low = readLowSurrogateEscape(); + outBytes = ensureStringDecodeCapacity(outBytes, out + 4); + out = putUtf16Char(outBytes, out, escaped); + out = putUtf16Char(outBytes, out, low); + } else if (Character.isLowSurrogate(escaped)) { + throw error("Unpaired low surrogate escape"); + } else { + outBytes = ensureStringDecodeCapacity(outBytes, out + 2); + out = putUtf16Char(outBytes, out, escaped); + } + } else if (ch < 0x20) { + throw error("Control character in string"); + } else if (Character.isHighSurrogate((char) ch)) { + char low = readRawLowSurrogate(); + outBytes = ensureStringDecodeCapacity(outBytes, out + 4); + out = putUtf16Char(outBytes, out, (char) ch); + out = putUtf16Char(outBytes, out, low); + } else if (Character.isLowSurrogate((char) ch)) { + throw error("Unpaired low surrogate in string"); + } else { + outBytes = ensureStringDecodeCapacity(outBytes, out + 2); + out = putUtf16Char(outBytes, out, (char) ch); + } + ch = nextStringChar(); + } + } + + private int nextStringChar() { + if (position >= length) { + throw error("Unterminated string"); + } + return charAtFast(position++); + } + + private char readRawLowSurrogate() { + if (position >= length) { + throw error("Unpaired high surrogate in string"); + } + char low = charAtFast(position++); + if (!Character.isLowSurrogate(low)) { + throw error("Unpaired high surrogate in string"); + } + return low; + } + + private char readEscapedStringChar() { + if (position >= length) { + throw error("Unterminated escape"); + } + char escaped = charAtFast(position++); + switch (escaped) { + case '"': + case '\\': + case '/': + return escaped; + case 'b': + return '\b'; + case 'f': + return '\f'; + case 'n': + return '\n'; + case 'r': + return '\r'; + case 't': + return '\t'; + case 'u': + return readUnicodeEscape(); + default: + throw error("Invalid escape"); + } + } + + private char readLowSurrogateEscape() { + if (position + 2 > length || charAtFast(position) != '\\' || charAtFast(position + 1) != 'u') { + throw error("Unpaired high surrogate escape"); + } + position += 2; + char low = readUnicodeEscape(); + if (!Character.isLowSurrogate(low)) { + throw error("Unpaired high surrogate escape"); + } + return low; + } + + private String finishDecodedString(byte[] outBytes, int length, boolean utf16) { + // The decode buffer is reader-owned reusable storage; returned Strings must own exact bytes. + byte[] result = new byte[length]; + System.arraycopy(outBytes, 0, result, 0, length); + return utf16 + ? StringSerializer.newUtf16StringZeroCopy(result) + : StringSerializer.newLatin1StringZeroCopy(result); + } + + private byte[] ensureStringDecodeCapacity(byte[] outBytes, int capacity) { + if (outBytes.length < capacity) { + return growStringDecodeBuffer(outBytes, capacity); + } + return outBytes; + } + + private byte[] growStringDecodeBuffer(byte[] outBytes, int capacity) { + int newCapacity = Math.max(capacity, outBytes.length << 1); + byte[] grown = Arrays.copyOf(outBytes, newCapacity); + stringDecodeBuffer = grown; + return grown; + } + + private byte[] widenStringDecodeBuffer(byte[] outBytes, int length) { + int utf16Length = length << 1; + outBytes = ensureStringDecodeCapacity(outBytes, utf16Length); + for (int i = length - 1, pos = utf16Length - 2; i >= 0; i--, pos -= 2) { + putUtf16Char(outBytes, pos, (char) (outBytes[i] & 0xFF)); + } + return outBytes; + } + + private static int putUtf16Char(byte[] outBytes, int pos, char value) { + if (LITTLE_ENDIAN) { + outBytes[pos] = (byte) value; + outBytes[pos + 1] = (byte) (value >>> 8); + } else { + outBytes[pos] = (byte) (value >>> 8); + outBytes[pos + 1] = (byte) value; + } + return pos + 2; + } + + private String readStringLoop(int start, boolean nonLatin) { while (position < length) { char ch = charAtFast(position++); if (ch == '"') { - if (builder == null) { - return input.substring(start, position - 1); - } - builder.append(input, start, position - 1); - return builder.toString(); + return input.substring(start, position - 1); } else if (ch == '\\') { - if (builder == null) { - builder = new StringBuilder(); - } - builder.append(input, start, position - 1); - appendEscape(builder); - start = position; + return readEscapedStringTail(start, position - 1, nonLatin); } else if (ch < 0x20) { throw error("Control character in string"); } else if (Character.isHighSurrogate(ch)) { @@ -628,13 +1066,194 @@ private String readStringToken() { throw error("Unpaired high surrogate in string"); } position++; + nonLatin = true; } else if (Character.isLowSurrogate(ch)) { throw error("Unpaired low surrogate in string"); + } else if (ch > 0xFF) { + nonLatin = true; } } throw error("Unterminated string"); } + private LocalDate readIsoLocalDateToken() { + int offset = position; + int inputLength = length; + if (offset + 12 > inputLength || charAtFast(offset++) != '"') { + throw error("Expected string"); + } + int dateStart = offset; + if (charAtFast(dateStart + 4) != '-' || charAtFast(dateStart + 7) != '-') { + throw new IllegalArgumentException(); + } + int year = parse4(dateStart); + int month = parse2(dateStart + 5); + int day = parse2(dateStart + 8); + int end = dateStart + 10; + char ch = charAtFast(end); + if (ch == '"') { + position = end + 1; + return LocalDate.of(year, month, day); + } + if (ch == 'T') { + position = scanSimpleStringTail(end + 1); + return LocalDate.of(year, month, day); + } + throw new IllegalArgumentException(); + } + + private OffsetDateTime readIsoOffsetDateTimeToken() { + int offset = position; + int inputLength = length; + if (offset + 19 > inputLength || charAtFast(offset++) != '"') { + throw error("Expected string"); + } + int start = offset; + if (charAtFast(start + 4) != '-' + || charAtFast(start + 7) != '-' + || charAtFast(start + 10) != 'T' + || charAtFast(start + 13) != ':') { + throw new IllegalArgumentException(); + } + int year = parse4(start); + int month = parse2(start + 5); + int day = parse2(start + 8); + int hour = parse2(start + 11); + int minute = parse2(start + 14); + return readIsoOffsetDateTimeTail(start + 16, inputLength, year, month, day, hour, minute); + } + + private OffsetDateTime readIsoOffsetDateTimeTail( + int index, int inputLength, int year, int month, int day, int hour, int minute) { + int second = 0; + int nano = 0; + if (index < inputLength && charAtFast(index) == ':') { + second = parse2(index + 1); + index += 3; + if (index < inputLength && charAtFast(index) == '.') { + int fractionStart = index + 1; + int fractionEnd = fractionStart; + while (fractionEnd < inputLength && isDigit(charAtFast(fractionEnd))) { + fractionEnd++; + } + if (fractionEnd == fractionStart || fractionEnd - fractionStart > 9) { + throw new IllegalArgumentException(); + } + nano = parseNano(fractionStart, fractionEnd); + index = fractionEnd; + } + } + if (index < inputLength && charAtFast(index) == 'Z') { + if (index + 1 >= inputLength || charAtFast(index + 1) != '"') { + throw new IllegalArgumentException(); + } + position = index + 2; + return OffsetDateTime.of(year, month, day, hour, minute, second, nano, ZoneOffset.UTC); + } + return readIsoOffsetDateTimeOffsetTail( + index, inputLength, year, month, day, hour, minute, second, nano); + } + + private OffsetDateTime readIsoOffsetDateTimeOffsetTail( + int index, + int inputLength, + int year, + int month, + int day, + int hour, + int minute, + int second, + int nano) { + long offsetAndEnd = parseOffsetAndEnd(index, inputLength); + position = (int) offsetAndEnd; + return OffsetDateTime.of( + year, + month, + day, + hour, + minute, + second, + nano, + ZoneOffset.ofTotalSeconds((int) (offsetAndEnd >> 32))); + } + + private int scanSimpleStringTail(int offset) { + int inputLength = length; + while (offset < inputLength) { + char ch = charAtFast(offset++); + if (ch == '"') { + return offset; + } + if (ch == '\\' || ch < 0x20 || Character.isSurrogate(ch)) { + throw new IllegalArgumentException(); + } + } + throw error("Unterminated string"); + } + + private long parseOffsetAndEnd(int index, int inputLength) { + char offset = charAtFast(index); + if (offset == 'Z') { + if (index + 1 >= inputLength || charAtFast(index + 1) != '"') { + throw new IllegalArgumentException(); + } + return ((long) (index + 2)) & 0xFFFF_FFFFL; + } + if (offset != '+' && offset != '-') { + throw new IllegalArgumentException(); + } + if (index + 6 >= inputLength || charAtFast(index + 3) != ':') { + throw new IllegalArgumentException(); + } + int hour = parse2(index + 1); + int minute = parse2(index + 4); + int second = 0; + int end = index + 6; + if (charAtFast(end) == ':') { + if (end + 3 >= inputLength) { + throw new IllegalArgumentException(); + } + second = parse2(end + 1); + end += 3; + } + if (charAtFast(end) != '"') { + throw new IllegalArgumentException(); + } + int total = hour * 3600 + minute * 60 + second; + if (offset == '-') { + total = -total; + } + return ((long) total << 32) | ((long) (end + 1) & 0xFFFF_FFFFL); + } + + private int parseNano(int start, int end) { + int nano = 0; + for (int i = start; i < end; i++) { + nano = nano * 10 + charAtFast(i) - '0'; + } + for (int i = end - start; i < 9; i++) { + nano *= 10; + } + return nano; + } + + private int parse4(int index) { + return parse2(index) * 100 + parse2(index + 2); + } + + private int parse2(int index) { + int high = charAtFast(index) - '0'; + int low = charAtFast(index + 1) - '0'; + if (high < 0 || high > 9 || low < 0 || low > 9) { + throw new IllegalArgumentException(); + } + return high * 10 + low; + } + + private static boolean isDigit(char ch) { + return ch >= '0' && ch <= '9'; + } + @Override public JsonFieldInfo readField(JsonFieldTable table) { return table.get(readFieldNameHash()); @@ -666,7 +1285,7 @@ public boolean tryReadNextFieldNameColon( long expectedHash, long expectedMask, int expectedLength) { int mark = position; if (mark < length) { - char ch = charAtFast(mark); + int ch = asciiAtFast(mark); if (ch == '"') { return tryReadFieldNameColonAt(mark, expectedHash, expectedMask, expectedLength); } @@ -677,8 +1296,117 @@ public boolean tryReadNextFieldNameColon( return tryReadFieldNameColon(expectedHash, expectedMask, expectedLength); } + public boolean tryReadNextFieldNameUtf16( + long expectedHash, + long expectedMask, + long firstWord, + long firstMask, + long secondWord, + long secondMask, + int expectedLength) { + byte[] localBytes = bytes; + if (localBytes == null) { + return tryReadNextFieldNameColon(expectedHash, expectedMask, expectedLength); + } + int mark = position; + if (mark < length) { + if (utf16CharEquals(localBytes, mark << 1, '"')) { + return tryReadUtf16FieldNameAt( + localBytes, + mark, + expectedHash, + expectedLength, + firstWord, + firstMask, + secondWord, + secondMask); + } + if (!isWhitespace(asciiAtFast(mark))) { + return false; + } + } + skipWhitespaceFast(); + return tryReadUtf16FieldNameAt( + localBytes, + mark, + expectedHash, + expectedLength, + firstWord, + firstMask, + secondWord, + secondMask); + } + + public boolean tryReadNextFieldNameUtf16Token2( + long firstWord, long firstMask, long secondWord, long secondMask, int tokenLength) { + byte[] localBytes = bytes; + int mark = position; + int tokenEnd = mark + tokenLength; + if (localBytes != null && tokenEnd <= length) { + int byteOffset = mark << 1; + if ((LittleEndian.getInt64(localBytes, byteOffset) & firstMask) == firstWord + && (secondMask == 0 + || readUtf16TokenWord(localBytes, byteOffset + Long.BYTES, tokenLength - 4) + == secondWord)) { + position = tokenEnd; + return true; + } + } + return false; + } + + public boolean tryReadNextFieldNameUtf16Token3( + long firstWord, long secondWord, long thirdWord, int tokenLength) { + byte[] localBytes = bytes; + int mark = position; + int tokenEnd = mark + tokenLength; + if (localBytes != null && tokenEnd <= length) { + int byteOffset = mark << 1; + if (LittleEndian.getInt64(localBytes, byteOffset) == firstWord + && LittleEndian.getInt64(localBytes, byteOffset + Long.BYTES) == secondWord + && readUtf16TokenWord(localBytes, byteOffset + (Long.BYTES << 1), tokenLength - 8) + == thirdWord) { + position = tokenEnd; + return true; + } + } + return false; + } + + private static long readUtf16TokenWord(byte[] localBytes, int byteOffset, int chars) { + if (chars == 4 || byteOffset + Long.BYTES <= localBytes.length) { + return LittleEndian.getInt64(localBytes, byteOffset) & utf16WordMask(chars); + } + long word = 0; + for (int i = 0; i < chars; i++) { + int offset = byteOffset + (i << 1); + long ch = (localBytes[offset] & 0xFFL) | ((localBytes[offset + 1] & 0xFFL) << 8); + word |= ch << (i << 4); + } + return word; + } + private boolean tryReadFieldNameColonAt( int mark, long expectedHash, long expectedMask, int expectedLength) { + byte[] localBytes = bytes; + if (localBytes != null + && expectedLength > 0 + && expectedLength <= Long.BYTES + && isJsonPackedName(expectedHash, expectedMask)) { + return tryReadUtf16FieldNameAt( + localBytes, + mark, + expectedHash, + expectedLength, + packedUtf16Word(expectedHash), + utf16WordMask(Math.min(expectedLength, 4)), + expectedLength > 4 ? packedUtf16Word(expectedHash >>> 32) : 0, + expectedLength > 4 ? utf16WordMask(expectedLength - 4) : 0); + } + return tryReadFieldNameColonLoop(mark, expectedHash, expectedLength); + } + + private boolean tryReadFieldNameColonLoop(int mark, long expectedHash, int expectedLength) { int offset = position; int end = offset + expectedLength + 1; if (end < length && charAtFast(offset++) == '"') { @@ -705,6 +1433,91 @@ private boolean tryReadFieldNameColonAt( return false; } + private boolean tryReadUtf16FieldNameAt( + byte[] localBytes, + int mark, + long expectedHash, + int expectedLength, + long firstWord, + long firstMask, + long secondWord, + long secondMask) { + int offset = position; + int quoteOffset = offset + expectedLength + 1; + int nameOffset = (offset + 1) << 1; + if ((nameOffset + Long.BYTES > localBytes.length) + || (secondMask != 0 && nameOffset + (Long.BYTES << 1) > localBytes.length)) { + return tryReadFieldNameColonLoop(mark, expectedHash, expectedLength); + } + if (quoteOffset < length + && utf16CharEquals(localBytes, offset << 1, '"') + && (LittleEndian.getInt64(localBytes, nameOffset) & firstMask) == firstWord + && (secondMask == 0 + || (LittleEndian.getInt64(localBytes, nameOffset + Long.BYTES) & secondMask) + == secondWord) + && utf16CharEquals(localBytes, quoteOffset << 1, '"')) { + int colonOffset = quoteOffset + 1; + if (colonOffset < length && utf16CharEquals(localBytes, colonOffset << 1, ':')) { + position = colonOffset + 1; + } else { + readFieldNameColon(colonOffset); + } + return true; + } + position = mark; + return false; + } + + private static boolean isJsonPackedName(long packed, long mask) { + long value = (packed & mask) | (~mask & CONTROL_LIMIT_BYTES); + long control = (value - CONTROL_LIMIT_BYTES) & ~value & BYTE_HIGH_BITS; + return (control | byteMatchMask(value, QUOTE_BYTES) | byteMatchMask(value, BACKSLASH_BYTES)) + == 0; + } + + private static long byteMatchMask(long word, long repeatedByte) { + long match = word ^ repeatedByte; + return (match - BYTE_ONES) & ~match & BYTE_HIGH_BITS; + } + + private static long utf16StringStopMask(long word, long nonLatinBytes) { + long syntaxStop = + utf16CharMatchMask(word, UTF16_QUOTE_CHARS) + | utf16CharMatchMask(word, UTF16_BACKSLASH_CHARS); + long controlStop = (word - UTF16_CONTROL_LIMIT) & ~word & UTF16_HIGH_BITS; + long stop = syntaxStop | controlStop; + if (nonLatinBytes != 0) { + stop |= utf16CharMatchMask(word & UTF16_SURROGATE_MASK, UTF16_SURROGATE_PREFIX); + } + return stop; + } + + private static long utf16CharMatchMask(long word, long repeatedChar) { + long match = word ^ repeatedChar; + return (match - UTF16_ONES) & ~match & UTF16_HIGH_BITS; + } + + private static long packedUtf16Word(long packed) { + long word = spreadLatin1ToUtf16(packed & 0xFFFFFFFFL); + return LITTLE_ENDIAN ? word : word << 8; + } + + private static long utf16WordMask(int length) { + return length == 4 ? -1L : (1L << (length << 4)) - 1; + } + + private static long spreadLatin1ToUtf16(long value) { + value = (value | (value << 16)) & UTF16_PAIR_MASK; + return (value | (value << 8)) & UTF16_BYTE_MASK; + } + + private static boolean utf16CharEquals(byte[] localBytes, int byteOffset, int expected) { + if (LITTLE_ENDIAN) { + return (localBytes[byteOffset] & 0xFF) == expected && localBytes[byteOffset + 1] == 0; + } + return localBytes[byteOffset] == 0 && (localBytes[byteOffset + 1] & 0xFF) == expected; + } + private void readFieldNameColon(int colonOffset) { position = colonOffset; expectNextToken(':'); @@ -856,7 +1669,7 @@ protected String slice(int start, int end) { private void skipWhitespaceFast() { int inputLength = length; while (position < inputLength) { - char ch = charAtFast(position); + int ch = asciiAtFast(position); if (isWhitespace(ch)) { position++; } else { @@ -865,7 +1678,7 @@ private void skipWhitespaceFast() { } } - private static boolean isWhitespace(char ch) { + private static boolean isWhitespace(int ch) { return ch == ' ' || ch == '\n' || ch == '\r' || ch == '\t'; } @@ -944,4 +1757,17 @@ private char charAtFast(int index) { ? input.charAt(index) : StringSerializer.getBytesChar(localBytes, index << 1); } + + private int asciiAtFast(int index) { + byte[] localBytes = bytes; + if (localBytes == null) { + return input.charAt(index); + } + // JSON structural tokens are ASCII; non-ASCII code units cannot match them or whitespace. + int byteOffset = index << 1; + if (LITTLE_ENDIAN) { + return localBytes[byteOffset + 1] == 0 ? localBytes[byteOffset] & 0xff : 0x100; + } + return localBytes[byteOffset] == 0 ? localBytes[byteOffset + 1] & 0xff : 0x100; + } } diff --git a/java/fory-json/src/main/java/org/apache/fory/json/reader/Utf8JsonReader.java b/java/fory-json/src/main/java/org/apache/fory/json/reader/Utf8JsonReader.java index df345945ea..d85a2abbfa 100644 --- a/java/fory-json/src/main/java/org/apache/fory/json/reader/Utf8JsonReader.java +++ b/java/fory-json/src/main/java/org/apache/fory/json/reader/Utf8JsonReader.java @@ -19,14 +19,24 @@ package org.apache.fory.json.reader; +import java.math.BigDecimal; +import java.time.LocalDate; +import java.time.OffsetDateTime; +import java.time.ZoneOffset; +import java.util.Arrays; +import java.util.UUID; import org.apache.fory.json.meta.JsonFieldInfo; import org.apache.fory.json.meta.JsonFieldNameHash; import org.apache.fory.json.meta.JsonFieldTable; import org.apache.fory.memory.LittleEndian; +import org.apache.fory.memory.NativeByteOrder; import org.apache.fory.serializer.StringSerializer; public final class Utf8JsonReader extends JsonReader { private static final byte[] EMPTY_BYTES = new byte[0]; + private static final int INITIAL_STRING_DECODE_BUFFER_SIZE = 1024; + private static final int RETAINED_STRING_DECODE_BUFFER_SIZE = 8192; + private static final boolean LITTLE_ENDIAN = NativeByteOrder.IS_LITTLE_ENDIAN; private static final long BYTE_ONES = 0x0101010101010101L; private static final int INT_BYTE_ONES = 0x01010101; private static final long BYTE_HIGH_BITS = 0x8080808080808080L; @@ -37,10 +47,21 @@ public final class Utf8JsonReader extends JsonReader { private static final int INT_CONTROL_LIMIT_BYTES = 0x20202020; private static final long QUOTE_BYTES = 0x2222222222222222L; private static final int INT_QUOTE_BYTES = 0x22222222; + private static final long LONG_MAX_DIV_10 = Long.MAX_VALUE / 10; + private static final int LONG_MAX_MOD_10 = (int) (Long.MAX_VALUE % 10); + private static final long LONG_MAX_DIV_100 = Long.MAX_VALUE / 100; + private static final int LONG_MAX_MOD_100 = (int) (Long.MAX_VALUE % 100); + private static final long LONG_MIN_DIV_10 = Long.MIN_VALUE / 10; + private static final int LONG_MIN_LAST_DIGIT = (int) -(Long.MIN_VALUE % 10); + private static final long EIGHT_DIGITS = 100_000_000L; + private static final long ASCII_ZEROES = 0x3030_3030_3030_3030L; + private static final long ASCII_NINES = 0x3939_3939_3939_3939L; + private static final long ASCII_HIGH_BITS = 0x8080_8080_8080_8080L; // JSON syntax bytes are ASCII, so hot token checks can compare signed bytes directly. // UTF-8 string decoding must keep unsigned byte conversion for non-ASCII content. private byte[] input; + private byte[] stringDecodeBuffer = new byte[INITIAL_STRING_DECODE_BUFFER_SIZE]; public Utf8JsonReader() { input = EMPTY_BYTES; @@ -59,6 +80,9 @@ public Utf8JsonReader reset(byte[] input) { public void clear() { input = EMPTY_BYTES; position = 0; + if (stringDecodeBuffer.length > RETAINED_STRING_DECODE_BUFFER_SIZE) { + stringDecodeBuffer = new byte[RETAINED_STRING_DECODE_BUFFER_SIZE]; + } } public boolean consumeToken(char expected) { @@ -103,13 +127,15 @@ public boolean consumeNextCommaOrEndObject() { position++; return true; } - if (ch == '}') { - position++; - return false; - } - if (!isWhitespace(ch)) { - return consumeNextCommaOrEndObjectSlow(); - } + return consumeNextObjectEndOrSlow(ch); + } + return consumeNextCommaOrEndObjectSlow(); + } + + private boolean consumeNextObjectEndOrSlow(int ch) { + if (ch == '}') { + position++; + return false; } return consumeNextCommaOrEndObjectSlow(); } @@ -137,13 +163,15 @@ public boolean consumeNextCommaOrEndArray() { position++; return true; } - if (ch == ']') { - position++; - return false; - } - if (!isWhitespace(ch)) { - return consumeNextCommaOrEndArraySlow(); - } + return consumeNextArrayEndOrSlow(ch); + } + return consumeNextCommaOrEndArraySlow(); + } + + private boolean consumeNextArrayEndOrSlow(int ch) { + if (ch == ']') { + position++; + return false; } return consumeNextCommaOrEndArraySlow(); } @@ -175,7 +203,7 @@ public boolean tryReadNextNullToken() { if (ch == 'n') { return tryReadNullLiteral(); } - if (!isWhitespace(ch)) { + if (ch > ' ' || !isWhitespace(ch)) { return false; } } @@ -183,8 +211,14 @@ public boolean tryReadNextNullToken() { } private boolean tryReadNullLiteral() { - if (startsWithAscii("null")) { - position += 4; + byte[] bytes = input; + int offset = position; + if (offset + 3 < bytes.length + && bytes[offset] == 'n' + && bytes[offset + 1] == 'u' + && bytes[offset + 2] == 'l' + && bytes[offset + 3] == 'l') { + position = offset + 4; return true; } return false; @@ -196,8 +230,11 @@ public boolean readBooleanValue() { } public boolean readNextBooleanValue() { - if (position < input.length && !isWhitespace(input[position])) { - return readBooleanToken(); + if (position < input.length) { + int ch = input[position]; + if (ch > ' ' || !isWhitespace(ch)) { + return readBooleanToken(); + } } return readBooleanValue(); } @@ -207,11 +244,22 @@ public boolean readBooleanTokenValue() { } private boolean readBooleanToken() { - if (startsWithAscii("true")) { - position += 4; + byte[] bytes = input; + int offset = position; + if (offset + 3 < bytes.length + && bytes[offset] == 't' + && bytes[offset + 1] == 'r' + && bytes[offset + 2] == 'u' + && bytes[offset + 3] == 'e') { + position = offset + 4; return true; - } else if (startsWithAscii("false")) { - position += 5; + } else if (offset + 4 < bytes.length + && bytes[offset] == 'f' + && bytes[offset + 1] == 'a' + && bytes[offset + 2] == 'l' + && bytes[offset + 3] == 's' + && bytes[offset + 4] == 'e') { + position = offset + 5; return false; } throw error("Expected boolean"); @@ -223,8 +271,11 @@ public int readIntValue() { } public int readNextIntValue() { - if (position < input.length && !isWhitespace(input[position])) { - return readIntToken(); + if (position < input.length) { + int ch = input[position]; + if (ch > ' ' || !isWhitespace(ch)) { + return readIntToken(); + } } return readIntValue(); } @@ -341,8 +392,11 @@ public long readLongValue() { } public long readNextLongValue() { - if (position < input.length && !isWhitespace(input[position])) { - return readLongToken(); + if (position < input.length) { + int ch = input[position]; + if (ch > ' ' || !isWhitespace(ch)) { + return readLongToken(); + } } return readLongValue(); } @@ -351,6 +405,42 @@ public long readLongTokenValue() { return readLongToken(); } + public BigDecimal readBigDecimal() { + skipWhitespaceFast(); + return readBigDecimalToken(); + } + + public UUID readUuid() { + skipWhitespaceFast(); + int mark = position; + try { + return readUuidToken(); + } catch (RuntimeException e) { + position = mark; + return UUID.fromString(readStringToken()); + } + } + + @Override + public double readDouble() { + skipWhitespaceFast(); + return readDoubleToken(); + } + + public double readNextDoubleValue() { + if (position < input.length) { + int ch = input[position]; + if (ch > ' ' || !isWhitespace(ch)) { + return readDoubleToken(); + } + } + return readDouble(); + } + + public double readDoubleTokenValue() { + return readDoubleToken(); + } + private long readLongToken() { byte[] bytes = input; int offset = position; @@ -377,6 +467,16 @@ private long readLongToken() { if (safeEnd > inputLength) { safeEnd = inputLength; } + int block = parseEightDigits(bytes, offset, safeEnd); + if (block >= 0) { + result = result * EIGHT_DIGITS + block; + offset += 8; + block = parseEightDigits(bytes, offset, safeEnd); + if (block >= 0) { + result = result * EIGHT_DIGITS + block; + offset += 8; + } + } while (offset < safeEnd) { ch = bytes[offset]; if (ch < '0' || ch > '9') { @@ -403,7 +503,7 @@ private long readPositiveLongTail(byte[] bytes, int offset, int inputLength, lon break; } int digit = ch - '0'; - if (result > (Long.MAX_VALUE - digit) / 10) { + if (result > LONG_MAX_DIV_10 || (result == LONG_MAX_DIV_10 && digit > LONG_MAX_MOD_10)) { position = offset; throw error("Long overflow"); } @@ -416,15 +516,15 @@ private long readPositiveLongTail(byte[] bytes, int offset, int inputLength, lon } private long readNegativeLongToken(int start) { - position = start + 1; - long result = 0; - long limit = Long.MIN_VALUE; - if (position >= input.length) { + byte[] bytes = input; + int offset = start + 1; + int inputLength = bytes.length; + if (offset >= inputLength) { throw error("Expected digit"); } - int ch = input[position]; + int ch = bytes[offset]; if (ch == '0') { - position++; + position = offset + 1; rejectLeadingDigitFast(); rejectFractionOrExponentFast(); return 0; @@ -432,27 +532,465 @@ private long readNegativeLongToken(int start) { if (ch < '1' || ch > '9') { throw error("Expected digit"); } - long multmin = limit / 10; - while (position < input.length) { - ch = input[position]; + long result = '0' - ch; + offset++; + int safeEnd = offset + 17; + if (safeEnd > inputLength) { + safeEnd = inputLength; + } + int block = parseEightDigits(bytes, offset, safeEnd); + if (block >= 0) { + result = result * EIGHT_DIGITS - block; + offset += 8; + block = parseEightDigits(bytes, offset, safeEnd); + if (block >= 0) { + result = result * EIGHT_DIGITS - block; + offset += 8; + } + } + while (offset < safeEnd) { + ch = bytes[offset]; if (ch < '0' || ch > '9') { break; } - int digit = ch - '0'; - if (result < multmin) { - throw error("Long overflow"); + result = result * 10 - (ch - '0'); + offset++; + } + if (offset < inputLength) { + ch = bytes[offset]; + if (ch >= '0' && ch <= '9') { + return readNegativeLongTail(bytes, offset, inputLength, result); } - result *= 10; - if (result < Long.MIN_VALUE + digit) { + } + position = offset; + rejectFractionOrExponentFast(); + return result; + } + + private long readNegativeLongTail(byte[] bytes, int offset, int inputLength, long result) { + while (offset < inputLength) { + int ch = bytes[offset]; + if (ch < '0' || ch > '9') { + break; + } + int digit = ch - '0'; + if (result < LONG_MIN_DIV_10 || (result == LONG_MIN_DIV_10 && digit > LONG_MIN_LAST_DIGIT)) { + position = offset; throw error("Long overflow"); } - result -= digit; - position++; + result = result * 10 - digit; + offset++; } + position = offset; rejectFractionOrExponentFast(); return result; } + private static int parseEightDigits(byte[] bytes, int offset, int safeEnd) { + if (offset + 8 > safeEnd) { + return -1; + } + // Keep this as one unaligned little-endian load. Eight separate byte loads made the helper too + // large for C2 to place well under generated readers, while the byte-lane math stays compact. + long chunk = LittleEndian.getInt64(bytes, offset); + long digits = chunk - ASCII_ZEROES; + if (((digits | (ASCII_NINES - chunk)) & ASCII_HIGH_BITS) != 0) { + return -1; + } + long pairs = (digits * 10 + (digits >>> 8)) & 0x00FF_00FF_00FF_00FFL; + long quads = (pairs * 100 + (pairs >>> 16)) & 0x0000_FFFF_0000_FFFFL; + return (int) ((quads & 0xFFFF) * 10_000 + (quads >>> 32)); + } + + private BigDecimal readBigDecimalToken() { + byte[] bytes = input; + int offset = position; + int start = offset; + int inputLength = bytes.length; + if (offset >= inputLength) { + return readBigDecimalFallback(start); + } + int ch = bytes[offset]; + if (ch == '-') { + return readSignedBigDecimalToken(start); + } + long unscaled = 0; + int scale = 0; + if (ch == '0') { + offset++; + position = offset; + rejectLeadingDigitFast(); + } else if (ch >= '1' && ch <= '9') { + do { + int digit = ch - '0'; + if (unscaled > (Long.MAX_VALUE - digit) / 10) { + return readBigDecimalFallback(start); + } + unscaled = unscaled * 10 + digit; + offset++; + if (offset >= inputLength) { + break; + } + ch = bytes[offset]; + } while (ch >= '0' && ch <= '9'); + } else { + return readBigDecimalFallback(start); + } + if (offset < inputLength && bytes[offset] == '.') { + offset++; + int fractionStart = offset; + while (offset < inputLength) { + ch = bytes[offset]; + if (ch < '0' || ch > '9') { + break; + } + int digit = ch - '0'; + if (unscaled > (Long.MAX_VALUE - digit) / 10) { + return readBigDecimalFallback(start); + } + unscaled = unscaled * 10 + digit; + scale++; + offset++; + } + if (offset == fractionStart) { + return readBigDecimalFallback(start); + } + } + if (offset < inputLength) { + ch = bytes[offset]; + if (ch == 'e' || ch == 'E') { + return readBigDecimalFallback(start); + } + } + position = offset; + return BigDecimal.valueOf(unscaled, scale); + } + + private BigDecimal readSignedBigDecimalToken(int start) { + byte[] bytes = input; + int offset = start + 1; + int inputLength = bytes.length; + if (offset >= inputLength) { + return readBigDecimalFallback(start); + } + int ch = bytes[offset]; + long unscaled = 0; + int scale = 0; + if (ch == '0') { + offset++; + position = offset; + rejectLeadingDigitFast(); + } else if (ch >= '1' && ch <= '9') { + do { + int digit = ch - '0'; + if (unscaled > (Long.MAX_VALUE - digit) / 10) { + return readBigDecimalFallback(start); + } + unscaled = unscaled * 10 + digit; + offset++; + if (offset >= inputLength) { + break; + } + ch = bytes[offset]; + } while (ch >= '0' && ch <= '9'); + } else { + return readBigDecimalFallback(start); + } + if (offset < inputLength && bytes[offset] == '.') { + offset++; + int fractionStart = offset; + while (offset < inputLength) { + ch = bytes[offset]; + if (ch < '0' || ch > '9') { + break; + } + int digit = ch - '0'; + if (unscaled > (Long.MAX_VALUE - digit) / 10) { + return readBigDecimalFallback(start); + } + unscaled = unscaled * 10 + digit; + scale++; + offset++; + } + if (offset == fractionStart) { + return readBigDecimalFallback(start); + } + } + if (offset < inputLength) { + ch = bytes[offset]; + if (ch == 'e' || ch == 'E') { + return readBigDecimalFallback(start); + } + } + position = offset; + return BigDecimal.valueOf(-unscaled, scale); + } + + private BigDecimal readBigDecimalFallback(int start) { + // Keep overflow and exponent forms on the existing string constructor path so the fast path + // only owns decimals that fit exactly as long + scale. + position = start; + return new BigDecimal(readNumber()); + } + + private UUID readUuidToken() { + byte[] bytes = input; + int offset = position; + int start = offset + 1; + if (offset + 38 > bytes.length || bytes[offset] != '"') { + throw new IllegalArgumentException(); + } + if (bytes[start + 8] != '-' + || bytes[start + 13] != '-' + || bytes[start + 18] != '-' + || bytes[start + 23] != '-' + || bytes[start + 36] != '"') { + throw new IllegalArgumentException(); + } + long msb = parseHex(bytes, start, 8); + msb = (msb << 16) | parseHex(bytes, start + 9, 4); + msb = (msb << 16) | parseHex(bytes, start + 14, 4); + long lsb = parseHex(bytes, start + 19, 4); + lsb = (lsb << 48) | parseHex(bytes, start + 24, 12); + position = start + 37; + return new UUID(msb, lsb); + } + + private static long parseHex(byte[] bytes, int offset, int length) { + long value = 0; + for (int i = 0; i < length; i++) { + value = (value << 4) | hexValue(bytes[offset + i]); + } + return value; + } + + private static int hexValue(int ch) { + if (ch >= '0' && ch <= '9') { + return ch - '0'; + } + int lower = ch | 0x20; + if (lower >= 'a' && lower <= 'f') { + return lower - 'a' + 10; + } + throw new IllegalArgumentException(); + } + + private double readDoubleToken() { + // Keep the fast path exact: compact plain decimals convert through BigDecimal's long+scale + // path, while exponents, overflow, and longer precision stay on Java's full parser. The + // two-digit accumulator below only reduces parser loop work; it must keep the same overflow + // boundary and BigDecimal finish instead of accepting approximate double construction. + byte[] bytes = input; + int offset = position; + int inputLength = bytes.length; + if (offset >= inputLength) { + return readDoubleFallback(offset); + } + int ch = bytes[offset]; + if (ch == '-') { + return readSignedDoubleToken(offset); + } + return readPositiveDoubleToken(bytes, offset, inputLength, ch); + } + + private double readPositiveDoubleToken(byte[] bytes, int offset, int inputLength, int ch) { + int start = offset; + long unscaled = 0; + int scale = 0; + if (ch == '0') { + offset++; + if (offset < inputLength) { + ch = bytes[offset]; + if (ch >= '0' && ch <= '9') { + return readDoubleFallback(start); + } + } + } else if (ch >= '1' && ch <= '9') { + unscaled = ch - '0'; + offset++; + while (offset + 1 < inputLength) { + int high = bytes[offset] - '0'; + int low = bytes[offset + 1] - '0'; + if (high < 0 || high > 9 || low < 0 || low > 9) { + break; + } + int pair = high * 10 + low; + if (unscaled > LONG_MAX_DIV_100 + || (unscaled == LONG_MAX_DIV_100 && pair > LONG_MAX_MOD_100)) { + return readDoubleFallback(start); + } + unscaled = unscaled * 100 + pair; + offset += 2; + } + if (offset < inputLength) { + int digit = bytes[offset] - '0'; + if (digit >= 0 && digit <= 9) { + if (unscaled > LONG_MAX_DIV_10 + || (unscaled == LONG_MAX_DIV_10 && digit > LONG_MAX_MOD_10)) { + return readDoubleFallback(start); + } + unscaled = unscaled * 10 + digit; + offset++; + } + } + } else { + return readDoubleFallback(start); + } + if (offset < inputLength && bytes[offset] == '.') { + offset++; + int fractionStart = offset; + while (offset + 1 < inputLength) { + int high = bytes[offset] - '0'; + int low = bytes[offset + 1] - '0'; + if (high < 0 || high > 9 || low < 0 || low > 9) { + break; + } + int pair = high * 10 + low; + if (unscaled > LONG_MAX_DIV_100 + || (unscaled == LONG_MAX_DIV_100 && pair > LONG_MAX_MOD_100)) { + return readDoubleFallback(start); + } + unscaled = unscaled * 100 + pair; + scale += 2; + offset += 2; + } + if (offset < inputLength) { + int digit = bytes[offset] - '0'; + if (digit >= 0 && digit <= 9) { + if (unscaled > LONG_MAX_DIV_10 + || (unscaled == LONG_MAX_DIV_10 && digit > LONG_MAX_MOD_10)) { + return readDoubleFallback(start); + } + unscaled = unscaled * 10 + digit; + scale++; + offset++; + } + } + if (offset == fractionStart) { + return readDoubleFallback(start); + } + } + return finishDoubleToken(bytes, offset, inputLength, start, unscaled, scale); + } + + private double readSignedDoubleToken(int start) { + byte[] bytes = input; + int offset = start + 1; + int inputLength = bytes.length; + if (offset >= inputLength) { + return readDoubleFallback(start); + } + int ch = bytes[offset]; + long unscaled = 0; + int scale = 0; + if (ch == '0') { + offset++; + if (offset < inputLength) { + ch = bytes[offset]; + if (ch >= '0' && ch <= '9') { + return readDoubleFallback(start); + } + } + } else if (ch >= '1' && ch <= '9') { + unscaled = ch - '0'; + offset++; + while (offset + 1 < inputLength) { + int high = bytes[offset] - '0'; + int low = bytes[offset + 1] - '0'; + if (high < 0 || high > 9 || low < 0 || low > 9) { + break; + } + int pair = high * 10 + low; + if (unscaled > LONG_MAX_DIV_100 + || (unscaled == LONG_MAX_DIV_100 && pair > LONG_MAX_MOD_100)) { + return readDoubleFallback(start); + } + unscaled = unscaled * 100 + pair; + offset += 2; + } + if (offset < inputLength) { + int digit = bytes[offset] - '0'; + if (digit >= 0 && digit <= 9) { + if (unscaled > LONG_MAX_DIV_10 + || (unscaled == LONG_MAX_DIV_10 && digit > LONG_MAX_MOD_10)) { + return readDoubleFallback(start); + } + unscaled = unscaled * 10 + digit; + offset++; + } + } + } else { + return readDoubleFallback(start); + } + if (offset < inputLength && bytes[offset] == '.') { + offset++; + int fractionStart = offset; + while (offset + 1 < inputLength) { + int high = bytes[offset] - '0'; + int low = bytes[offset + 1] - '0'; + if (high < 0 || high > 9 || low < 0 || low > 9) { + break; + } + int pair = high * 10 + low; + if (unscaled > LONG_MAX_DIV_100 + || (unscaled == LONG_MAX_DIV_100 && pair > LONG_MAX_MOD_100)) { + return readDoubleFallback(start); + } + unscaled = unscaled * 100 + pair; + scale += 2; + offset += 2; + } + if (offset < inputLength) { + int digit = bytes[offset] - '0'; + if (digit >= 0 && digit <= 9) { + if (unscaled > LONG_MAX_DIV_10 + || (unscaled == LONG_MAX_DIV_10 && digit > LONG_MAX_MOD_10)) { + return readDoubleFallback(start); + } + unscaled = unscaled * 10 + digit; + scale++; + offset++; + } + } + if (offset == fractionStart) { + return readDoubleFallback(start); + } + } + return finishSignedDoubleToken(bytes, offset, inputLength, start, unscaled, scale); + } + + private double finishDoubleToken( + byte[] bytes, int offset, int inputLength, int start, long unscaled, int scale) { + if (offset < inputLength) { + int ch = bytes[offset]; + if (ch == 'e' || ch == 'E') { + return readDoubleFallback(start); + } + } + position = offset; + return BigDecimal.valueOf(unscaled, scale).doubleValue(); + } + + private double finishSignedDoubleToken( + byte[] bytes, int offset, int inputLength, int start, long unscaled, int scale) { + if (offset < inputLength) { + int ch = bytes[offset]; + if (ch == 'e' || ch == 'E') { + return readDoubleFallback(start); + } + } + position = offset; + if (unscaled == 0) { + return -0.0d; + } + return BigDecimal.valueOf(-unscaled, scale).doubleValue(); + } + + private double readDoubleFallback(int start) { + position = start; + return Double.parseDouble(readNumber()); + } + @Override public int readFieldNameInt() { skipWhitespaceFast(); @@ -603,7 +1141,7 @@ public String readNextNullableString() { if (ch == 'n' && tryReadNullLiteral()) { return null; } - if (!isWhitespace(ch)) { + if (ch > ' ' || !isWhitespace(ch)) { return readStringToken(); } } @@ -623,6 +1161,28 @@ public String readNullableStringToken() { return readStringToken(); } + public LocalDate readIsoLocalDate() { + skipWhitespaceFast(); + int mark = position; + try { + return readIsoLocalDateToken(); + } catch (RuntimeException e) { + position = mark; + throw e; + } + } + + public OffsetDateTime readIsoOffsetDateTime() { + skipWhitespaceFast(); + int mark = position; + try { + return readIsoOffsetDateTimeToken(); + } catch (RuntimeException e) { + position = mark; + throw e; + } + } + private String readStringToken() { byte[] bytes = input; int inputLength = bytes.length; @@ -631,6 +1191,67 @@ private String readStringToken() { } int start = position; int offset = start; + if (offset + Long.BYTES <= inputLength) { + long stopMask = stringStopMask(LittleEndian.getInt64(bytes, offset)); + if (stopMask != 0) { + return readStringWordStop(start, offset, stopMask); + } + offset += Long.BYTES; + if (offset + Long.BYTES <= inputLength) { + stopMask = stringStopMask(LittleEndian.getInt64(bytes, offset)); + if (stopMask != 0) { + return readStringWordStop(start, offset, stopMask); + } + offset += Long.BYTES; + if (offset + Long.BYTES <= inputLength) { + stopMask = stringStopMask(LittleEndian.getInt64(bytes, offset)); + if (stopMask != 0) { + return readStringWordStop(start, offset, stopMask); + } + offset += Long.BYTES; + } + } + } + return readStringTokenLongTail(start, offset, inputLength); + } + + private String readStringWordStop(int start, int offset, long stopMask) { + int stop = offset + (Long.numberOfTrailingZeros(stopMask) >>> 3); + int b = input[stop]; + if (b == '"') { + position = stop + 1; + return newLatin1String(start, stop); + } + return readStringStop(start, stop, b); + } + + private String readStringTokenLongTail(int start, int offset, int inputLength) { + byte[] bytes = input; + int doubleWordEnd = inputLength - (Long.BYTES << 1); + while (offset <= doubleWordEnd) { + long stopMask = stringStopMask(LittleEndian.getInt64(bytes, offset)); + if (stopMask != 0) { + int stop = offset + (Long.numberOfTrailingZeros(stopMask) >>> 3); + int b = bytes[stop]; + if (b == '"') { + position = stop + 1; + return newLatin1String(start, stop); + } + return readStringStop(start, stop, b); + } + int nextOffset = offset + Long.BYTES; + stopMask = stringStopMask(LittleEndian.getInt64(bytes, nextOffset)); + if (stopMask != 0) { + int stop = nextOffset + (Long.numberOfTrailingZeros(stopMask) >>> 3); + int b = bytes[stop]; + if (b == '"') { + position = stop + 1; + return newLatin1String(start, stop); + } + return readStringStop(start, stop, b); + } + offset = nextOffset + Long.BYTES; + } int wordEnd = inputLength - Long.BYTES; while (offset <= wordEnd) { long stopMask = stringStopMask(LittleEndian.getInt64(bytes, offset)); @@ -646,6 +1267,13 @@ private String readStringToken() { } return readStringStop(start, stop, b); } + return readStringTokenTail(start, offset, inputLength); + } + + private String readStringTokenTail(int start, int offset, int inputLength) { + byte[] bytes = input; + // Reached only when the whole input has fewer than eight bytes left. Keep this rare tail out + // of the hot word scanner so C2 has more budget for common string call sites. if (offset + Integer.BYTES <= inputLength) { int stopMask = stringStopMask(LittleEndian.getInt32(bytes, offset)); if (stopMask == 0) { @@ -680,51 +1308,217 @@ private String readStringToken() { throw error("Unterminated string"); } - private String readStringStop(int start, int stop, int b) { - position = stop + 1; - if (b >= 0 && b < 0x20) { - throw error("Control character in string"); + private LocalDate readIsoLocalDateToken() { + byte[] bytes = input; + int offset = position; + int length = bytes.length; + if (offset + 12 > length || bytes[offset++] != '"') { + throw error("Expected string"); + } + int dateStart = offset; + if (bytes[dateStart + 4] != '-' || bytes[dateStart + 7] != '-') { + throw new IllegalArgumentException(); + } + int year = parse4(bytes, dateStart); + int month = parse2(bytes, dateStart + 5); + int day = parse2(bytes, dateStart + 8); + int end = dateStart + 10; + int ch = bytes[end]; + if (ch == '"') { + position = end + 1; + return LocalDate.of(year, month, day); } - StringBuilder builder = newStringBuilder(start, stop); - appendAscii(builder, start, stop); - if (b == '\\') { - appendEscape(builder); - return readStringTail(builder); + if (ch == 'T') { + position = scanSimpleStringTail(bytes, end + 1); + return LocalDate.of(year, month, day); } - if (b < 0) { - appendUtf8(builder, b & 0xFF); - return readStringTail(builder); + throw new IllegalArgumentException(); + } + + private OffsetDateTime readIsoOffsetDateTimeToken() { + byte[] bytes = input; + int offset = position; + int length = bytes.length; + if (offset + 19 > length || bytes[offset++] != '"') { + throw error("Expected string"); + } + int start = offset; + if (bytes[start + 4] != '-' + || bytes[start + 7] != '-' + || bytes[start + 10] != 'T' + || bytes[start + 13] != ':') { + throw new IllegalArgumentException(); + } + int year = parse4(bytes, start); + int month = parse2(bytes, start + 5); + int day = parse2(bytes, start + 8); + int hour = parse2(bytes, start + 11); + int minute = parse2(bytes, start + 14); + return readIsoOffsetDateTimeTail(bytes, start + 16, length, year, month, day, hour, minute); + } + + private OffsetDateTime readIsoOffsetDateTimeTail( + byte[] bytes, int index, int length, int year, int month, int day, int hour, int minute) { + int second = 0; + int nano = 0; + if (index < length && bytes[index] == ':') { + second = parse2(bytes, index + 1); + index += 3; + if (index < length && bytes[index] == '.') { + int fractionStart = index + 1; + int fractionEnd = fractionStart; + while (fractionEnd < length && isDigit(bytes[fractionEnd])) { + fractionEnd++; + } + if (fractionEnd == fractionStart || fractionEnd - fractionStart > 9) { + throw new IllegalArgumentException(); + } + nano = parseNano(bytes, fractionStart, fractionEnd); + index = fractionEnd; + } + } + if (index < length && bytes[index] == 'Z') { + if (index + 1 >= length || bytes[index + 1] != '"') { + throw new IllegalArgumentException(); + } + position = index + 2; + return OffsetDateTime.of(year, month, day, hour, minute, second, nano, ZoneOffset.UTC); + } + return readIsoOffsetDateTimeOffsetTail( + bytes, index, length, year, month, day, hour, minute, second, nano); + } + + private OffsetDateTime readIsoOffsetDateTimeOffsetTail( + byte[] bytes, + int index, + int length, + int year, + int month, + int day, + int hour, + int minute, + int second, + int nano) { + long offsetAndEnd = parseOffsetAndEnd(bytes, index, length); + position = (int) offsetAndEnd; + return OffsetDateTime.of( + year, + month, + day, + hour, + minute, + second, + nano, + ZoneOffset.ofTotalSeconds((int) (offsetAndEnd >> 32))); + } + + private int scanSimpleStringTail(byte[] bytes, int offset) { + int length = bytes.length; + while (offset < length) { + int b = bytes[offset++]; + if (b == '"') { + return offset; + } + if (b == '\\' || b < 0x20 || b < 0) { + throw new IllegalArgumentException(); + } } - appendUtf8(builder, b); - return readStringTail(builder); + throw error("Unterminated string"); } - private StringBuilder newStringBuilder(int start, int stop) { - return new StringBuilder(Math.max(16, stop - start + 16)); + private static long parseOffsetAndEnd(byte[] bytes, int index, int length) { + int offset = bytes[index]; + if (offset == 'Z') { + if (index + 1 >= length || bytes[index + 1] != '"') { + throw new IllegalArgumentException(); + } + return ((long) (index + 2)) & 0xFFFF_FFFFL; + } + if (offset != '+' && offset != '-') { + throw new IllegalArgumentException(); + } + if (index + 6 >= length || bytes[index + 3] != ':') { + throw new IllegalArgumentException(); + } + int hour = parse2(bytes, index + 1); + int minute = parse2(bytes, index + 4); + int second = 0; + int end = index + 6; + if (bytes[end] == ':') { + if (end + 3 >= length) { + throw new IllegalArgumentException(); + } + second = parse2(bytes, end + 1); + end += 3; + } + if (bytes[end] != '"') { + throw new IllegalArgumentException(); + } + int total = hour * 3600 + minute * 60 + second; + if (offset == '-') { + total = -total; + } + return ((long) total << 32) | ((long) (end + 1) & 0xFFFF_FFFFL); } - private static long stringStopMask(long word) { - return (word & BYTE_HIGH_BITS) - | byteMatchMask(word, QUOTE_BYTES) - | byteMatchMask(word, BACKSLASH_BYTES) - | ((word - CONTROL_LIMIT_BYTES) & ~word & BYTE_HIGH_BITS); + private static int parseNano(byte[] bytes, int start, int end) { + int nano = 0; + for (int i = start; i < end; i++) { + nano = nano * 10 + bytes[i] - '0'; + } + for (int i = end - start; i < 9; i++) { + nano *= 10; + } + return nano; } - private static int stringStopMask(int word) { - return (word & INT_BYTE_HIGH_BITS) - | byteMatchMask(word, INT_QUOTE_BYTES) - | byteMatchMask(word, INT_BACKSLASH_BYTES) - | ((word - INT_CONTROL_LIMIT_BYTES) & ~word & INT_BYTE_HIGH_BITS); + private static int parse4(byte[] bytes, int index) { + return parse2(bytes, index) * 100 + parse2(bytes, index + 2); } - private static long byteMatchMask(long word, long repeatedByte) { - long match = word ^ repeatedByte; - return (match - BYTE_ONES) & ~match & BYTE_HIGH_BITS; + private static int parse2(byte[] bytes, int index) { + int high = bytes[index] - '0'; + int low = bytes[index + 1] - '0'; + if (high < 0 || high > 9 || low < 0 || low > 9) { + throw new IllegalArgumentException(); + } + return high * 10 + low; } - private static int byteMatchMask(int word, int repeatedByte) { - int match = word ^ repeatedByte; - return (match - INT_BYTE_ONES) & ~match & INT_BYTE_HIGH_BITS; + private static boolean isDigit(byte b) { + return b >= '0' && b <= '9'; + } + + private String readStringStop(int start, int stop, int b) { + position = stop + 1; + int out = stop - start; + byte[] bytes = stringDecodeBuffer; + if (out == 0 && b < 0) { + int first = b & 0xFF; + if ((first & 0xF0) == 0xE0 || (first & 0xF8) == 0xF0) { + return readStringUtf16FromFirst(bytes, first); + } + } + if (bytes.length < out) { + bytes = growStringDecodeBuffer(bytes, out); + } + System.arraycopy(input, start, bytes, 0, out); + return readStringLatin1Tail(bytes, out, b); + } + + private static long stringStopMask(long word) { + // UTF-8 mode stops on every high-bit byte, and readStringToken only uses the first stop bit. + // Subtraction borrow may only create later high bits after an earlier real stop, so the + // compact syntax/range expression preserves the first-stop position. Latin1JsonReader cannot + // use this shortcut because high-bit Latin-1 bytes are valid string payload. + long syntaxStop = ((word ^ QUOTE_BYTES) - BYTE_ONES) | ((word ^ BACKSLASH_BYTES) - BYTE_ONES); + return (syntaxStop | word | (word - CONTROL_LIMIT_BYTES)) & BYTE_HIGH_BITS; + } + + private static int stringStopMask(int word) { + int syntaxStop = + ((word ^ INT_QUOTE_BYTES) - INT_BYTE_ONES) | ((word ^ INT_BACKSLASH_BYTES) - INT_BYTE_ONES); + return (syntaxStop | word | (word - INT_CONTROL_LIMIT_BYTES)) & INT_BYTE_HIGH_BITS; } @Override @@ -762,7 +1556,7 @@ public boolean tryReadNextFieldNameColon( if (ch == '"') { return tryReadFieldNameColonAt(mark, expectedHash, expectedMask, expectedLength); } - if (!isWhitespace(ch)) { + if (ch > ' ' || !isWhitespace(ch)) { return false; } } @@ -860,6 +1654,37 @@ private boolean tryReadNextRawToken3(long prefix, long prefixMask, int suffix, i return false; } + public boolean tryReadNextFieldNameToken8( + long prefix, long suffix, long suffixMask, int tokenLength) { + return tryReadNextRawToken8(prefix, suffix, suffixMask, tokenLength); + } + + private boolean tryReadNextRawToken8(long prefix, long suffix, long suffixMask, int tokenLength) { + byte[] bytes = input; + int mark = position; + int suffixOffset = mark + Long.BYTES; + if (mark + tokenLength <= bytes.length + && LittleEndian.getInt64(bytes, mark) == prefix + && readTokenSuffix(bytes, suffixOffset, tokenLength, suffixMask) == suffix) { + position = mark + tokenLength; + return true; + } + return false; + } + + private static long readTokenSuffix( + byte[] bytes, int suffixOffset, int tokenLength, long suffixMask) { + if (suffixOffset + Long.BYTES <= bytes.length) { + return LittleEndian.getInt64(bytes, suffixOffset) & suffixMask; + } + int suffixLength = tokenLength - Long.BYTES; + long suffix = 0; + for (int i = 0; i < suffixLength; i++) { + suffix |= (long) (bytes[suffixOffset + i] & 0xFF) << (i << 3); + } + return suffix; + } + private boolean tryReadFieldNameColonAt( int mark, long expectedHash, long expectedMask, int expectedLength) { byte[] bytes = input; @@ -923,8 +1748,11 @@ public long readPackedStringHash() { } public long readNextPackedStringHash() { - if (position < input.length && !isWhitespace(input[position])) { - return readPackedStringHashToken(); + if (position < input.length) { + int ch = input[position]; + if (ch > ' ' || !isWhitespace(ch)) { + return readPackedStringHashToken(); + } } return readPackedStringHash(); } @@ -1076,40 +1904,6 @@ private String newLatin1String(int start, int end) { return StringSerializer.newLatin1StringZeroCopy(bytes); } - private String readStringTail(StringBuilder builder) { - while (position < input.length) { - int b = input[position++] & 0xFF; - if (b == '"') { - return builder.toString(); - } else if (b == '\\') { - appendEscape(builder); - } else if (b < 0x20) { - throw error("Control character in string"); - } else if (b < 0x80) { - builder.append((char) b); - } else { - appendUtf8(builder, b); - } - } - throw error("Unterminated string"); - } - - private void appendAscii(StringBuilder builder, int start, int end) { - for (int i = start; i < end; i++) { - builder.append((char) (input[i] & 0xFF)); - } - } - - private void appendUtf8(StringBuilder builder, int first) { - int codePoint = readUtf8CodePoint(first); - if (codePoint <= 0xFFFF) { - builder.append((char) codePoint); - } else { - builder.append(Character.highSurrogate(codePoint)); - builder.append(Character.lowSurrogate(codePoint)); - } - } - private int readUtf8CodePoint(int first) { if ((first & 0xE0) == 0xC0) { int second = continuation(); @@ -1139,6 +1933,331 @@ private int readUtf8CodePoint(int first) { throw error("Invalid UTF-8 sequence"); } + private String readStringLatin1Tail(byte[] bytes, int out, int b) { + while (true) { + if (b == '"') { + return finishDecodedString(bytes, out, false); + } + if (b == '\\') { + char ch = readEscapedStringChar(); + if (Character.isHighSurrogate(ch)) { + char low = readLowSurrogateEscape(); + bytes = widenStringDecodeBuffer(bytes, out); + out <<= 1; + bytes = ensureStringDecodeCapacity(bytes, out + 4); + out = putUtf16Char(bytes, out, ch); + out = putUtf16Char(bytes, out, low); + return readStringUtf16Tail(bytes, out); + } + if (Character.isLowSurrogate(ch)) { + throw error("Unpaired low surrogate escape"); + } + if (ch <= 0xFF) { + bytes = ensureStringDecodeCapacity(bytes, out + 1); + bytes[out++] = (byte) ch; + } else { + bytes = widenStringDecodeBuffer(bytes, out); + out <<= 1; + bytes = ensureStringDecodeCapacity(bytes, out + 2); + out = putUtf16Char(bytes, out, ch); + return readStringUtf16Tail(bytes, out); + } + } else if (b >= 0 && b < 0x20) { + throw error("Control character in string"); + } else if (b >= 0 && b < 0x80) { + bytes = ensureStringDecodeCapacity(bytes, out + 1); + bytes[out++] = (byte) b; + } else { + int codePoint = readUtf8CodePoint(b & 0xFF); + if (codePoint <= 0xFF) { + bytes = ensureStringDecodeCapacity(bytes, out + 1); + bytes[out++] = (byte) codePoint; + } else { + bytes = widenStringDecodeBuffer(bytes, out); + out <<= 1; + if (codePoint <= 0xFFFF) { + bytes = ensureStringDecodeCapacity(bytes, out + 2); + out = putUtf16Char(bytes, out, (char) codePoint); + } else { + bytes = ensureStringDecodeCapacity(bytes, out + 4); + out = putUtf16Char(bytes, out, Character.highSurrogate(codePoint)); + out = putUtf16Char(bytes, out, Character.lowSurrogate(codePoint)); + } + return readStringUtf16Tail(bytes, out); + } + } + if (position >= input.length) { + throw error("Unterminated string"); + } + b = input[position++] & 0xFF; + } + } + + private String readStringUtf16Tail(byte[] bytes, int out) { + byte[] input = this.input; + int position = this.position; + int inputLength = input.length; + int capacity = bytes.length; + while (position < inputLength) { + int b = input[position++] & 0xFF; + if ((b & 0xF0) == 0xE0) { + if (position >= inputLength) { + this.position = position; + throw error("Short UTF-8 sequence"); + } + int second = input[position++] & 0xFF; + if ((second & 0xC0) != 0x80) { + this.position = position; + throw error("Invalid UTF-8 continuation"); + } + if (position >= inputLength) { + this.position = position; + throw error("Short UTF-8 sequence"); + } + int third = input[position++] & 0xFF; + if ((third & 0xC0) != 0x80) { + this.position = position; + throw error("Invalid UTF-8 continuation"); + } + int codePoint = ((b & 0x0F) << 12) | ((second & 0x3F) << 6) | (third & 0x3F); + if (codePoint < 0x800 || (codePoint >= 0xD800 && codePoint <= 0xDFFF)) { + this.position = position; + throw error("Invalid UTF-8 sequence"); + } + if (out + 2 > capacity) { + bytes = growStringDecodeBuffer(bytes, out + 2); + capacity = bytes.length; + } + out = putUtf16Char(bytes, out, (char) codePoint); + } else if (b == '"') { + this.position = position; + return finishDecodedString(bytes, out, true); + } else if (b == '\\') { + this.position = position; + char ch = readEscapedStringChar(); + position = this.position; + if (Character.isHighSurrogate(ch)) { + char low = readLowSurrogateEscape(); + position = this.position; + if (out + 4 > capacity) { + bytes = growStringDecodeBuffer(bytes, out + 4); + capacity = bytes.length; + } + out = putUtf16Char(bytes, out, ch); + out = putUtf16Char(bytes, out, low); + } else if (Character.isLowSurrogate(ch)) { + throw error("Unpaired low surrogate escape"); + } else { + if (out + 2 > capacity) { + bytes = growStringDecodeBuffer(bytes, out + 2); + capacity = bytes.length; + } + out = putUtf16Char(bytes, out, ch); + } + } else if (b < 0x20) { + this.position = position; + throw error("Control character in string"); + } else if (b < 0x80) { + if (out + 2 > capacity) { + bytes = growStringDecodeBuffer(bytes, out + 2); + capacity = bytes.length; + } + out = putUtf16Char(bytes, out, (char) b); + } else if ((b & 0xE0) == 0xC0) { + if (position >= inputLength) { + this.position = position; + throw error("Short UTF-8 sequence"); + } + int second = input[position++] & 0xFF; + if ((second & 0xC0) != 0x80) { + this.position = position; + throw error("Invalid UTF-8 continuation"); + } + int codePoint = ((b & 0x1F) << 6) | (second & 0x3F); + if (codePoint < 0x80) { + this.position = position; + throw error("Overlong UTF-8 sequence"); + } + if (out + 2 > capacity) { + bytes = growStringDecodeBuffer(bytes, out + 2); + capacity = bytes.length; + } + out = putUtf16Char(bytes, out, (char) codePoint); + } else if ((b & 0xF8) == 0xF0) { + if (position >= inputLength) { + this.position = position; + throw error("Short UTF-8 sequence"); + } + int second = input[position++] & 0xFF; + if ((second & 0xC0) != 0x80) { + this.position = position; + throw error("Invalid UTF-8 continuation"); + } + if (position >= inputLength) { + this.position = position; + throw error("Short UTF-8 sequence"); + } + int third = input[position++] & 0xFF; + if ((third & 0xC0) != 0x80) { + this.position = position; + throw error("Invalid UTF-8 continuation"); + } + if (position >= inputLength) { + this.position = position; + throw error("Short UTF-8 sequence"); + } + int fourth = input[position++] & 0xFF; + if ((fourth & 0xC0) != 0x80) { + this.position = position; + throw error("Invalid UTF-8 continuation"); + } + int codePoint = + ((b & 0x07) << 18) | ((second & 0x3F) << 12) | ((third & 0x3F) << 6) | (fourth & 0x3F); + if (codePoint < 0x10000 || codePoint > 0x10FFFF) { + this.position = position; + throw error("Invalid UTF-8 sequence"); + } + if (out + 4 > capacity) { + bytes = growStringDecodeBuffer(bytes, out + 4); + capacity = bytes.length; + } + out = putUtf16Char(bytes, out, Character.highSurrogate(codePoint)); + out = putUtf16Char(bytes, out, Character.lowSurrogate(codePoint)); + } else { + this.position = position; + throw error("Invalid UTF-8 sequence"); + } + } + throw error("Unterminated string"); + } + + private String readStringUtf16FromFirst(byte[] bytes, int first) { + int codePoint; + if ((first & 0xF0) == 0xE0) { + byte[] input = this.input; + int position = this.position; + int inputLength = input.length; + if (position >= inputLength) { + this.position = position; + throw error("Short UTF-8 sequence"); + } + int second = input[position++] & 0xFF; + if ((second & 0xC0) != 0x80) { + this.position = position; + throw error("Invalid UTF-8 continuation"); + } + if (position >= inputLength) { + this.position = position; + throw error("Short UTF-8 sequence"); + } + int third = input[position++] & 0xFF; + if ((third & 0xC0) != 0x80) { + this.position = position; + throw error("Invalid UTF-8 continuation"); + } + codePoint = ((first & 0x0F) << 12) | ((second & 0x3F) << 6) | (third & 0x3F); + if (codePoint < 0x800 || (codePoint >= 0xD800 && codePoint <= 0xDFFF)) { + this.position = position; + throw error("Invalid UTF-8 sequence"); + } + this.position = position; + } else { + codePoint = readUtf8CodePoint(first); + } + int out; + if (codePoint <= 0xFFFF) { + bytes = ensureStringDecodeCapacity(bytes, 2); + out = putUtf16Char(bytes, 0, (char) codePoint); + } else { + bytes = ensureStringDecodeCapacity(bytes, 4); + out = putUtf16Char(bytes, 0, Character.highSurrogate(codePoint)); + out = putUtf16Char(bytes, out, Character.lowSurrogate(codePoint)); + } + return readStringUtf16Tail(bytes, out); + } + + private char readEscapedStringChar() { + if (position >= input.length) { + throw error("Unterminated escape"); + } + char escaped = (char) (input[position++] & 0xFF); + switch (escaped) { + case '"': + case '\\': + case '/': + return escaped; + case 'b': + return '\b'; + case 'f': + return '\f'; + case 'n': + return '\n'; + case 'r': + return '\r'; + case 't': + return '\t'; + case 'u': + return readUnicodeEscape(); + default: + throw error("Invalid escape"); + } + } + + private char readLowSurrogateEscape() { + if (position + 2 > input.length || input[position] != '\\' || input[position + 1] != 'u') { + throw error("Unpaired high surrogate escape"); + } + position += 2; + char low = readUnicodeEscape(); + if (!Character.isLowSurrogate(low)) { + throw error("Unpaired high surrogate escape"); + } + return low; + } + + private String finishDecodedString(byte[] bytes, int length, boolean utf16) { + // Strings must not share the reader-owned decode buffer; the buffer is reused by later reads. + byte[] result = new byte[length]; + System.arraycopy(bytes, 0, result, 0, length); + return utf16 + ? StringSerializer.newUtf16StringZeroCopy(result) + : StringSerializer.newLatin1StringZeroCopy(result); + } + + private byte[] ensureStringDecodeCapacity(byte[] bytes, int capacity) { + if (bytes.length < capacity) { + return growStringDecodeBuffer(bytes, capacity); + } + return bytes; + } + + private byte[] growStringDecodeBuffer(byte[] bytes, int capacity) { + int newCapacity = Math.max(capacity, bytes.length << 1); + byte[] grown = Arrays.copyOf(bytes, newCapacity); + stringDecodeBuffer = grown; + return grown; + } + + private byte[] widenStringDecodeBuffer(byte[] bytes, int length) { + int utf16Length = length << 1; + bytes = ensureStringDecodeCapacity(bytes, utf16Length); + for (int i = length - 1, pos = utf16Length - 2; i >= 0; i--, pos -= 2) { + putUtf16Char(bytes, pos, (char) (bytes[i] & 0xFF)); + } + return bytes; + } + + private static int putUtf16Char(byte[] bytes, int pos, char value) { + if (LITTLE_ENDIAN) { + bytes[pos] = (byte) value; + bytes[pos + 1] = (byte) (value >>> 8); + } else { + bytes[pos] = (byte) (value >>> 8); + bytes[pos + 1] = (byte) value; + } + return pos + 2; + } + private int continuation() { if (position >= input.length) { throw error("Short UTF-8 sequence"); @@ -1153,6 +2272,9 @@ private int continuation() { private void skipWhitespaceFast() { while (position < input.length) { int ch = input[position]; + if (ch > ' ') { + return; + } if (isWhitespace(ch)) { position++; } else { @@ -1165,19 +2287,6 @@ private static boolean isWhitespace(int ch) { return ch == ' ' || ch == '\n' || ch == '\r' || ch == '\t'; } - private boolean startsWithAscii(String value) { - int end = position + value.length(); - if (end > input.length) { - return false; - } - for (int i = 0; i < value.length(); i++) { - if (input[position + i] != value.charAt(i)) { - return false; - } - } - return true; - } - private void rejectLeadingDigitFast() { if (position < input.length) { int ch = input[position]; diff --git a/java/fory-json/src/main/java/org/apache/fory/json/resolver/CodecRegistry.java b/java/fory-json/src/main/java/org/apache/fory/json/resolver/CodecRegistry.java index 629eae9d3f..e4623c0baa 100644 --- a/java/fory-json/src/main/java/org/apache/fory/json/resolver/CodecRegistry.java +++ b/java/fory-json/src/main/java/org/apache/fory/json/resolver/CodecRegistry.java @@ -19,6 +19,9 @@ package org.apache.fory.json.resolver; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -54,4 +57,18 @@ public CodecRegistry copy() { } return new CodecRegistry(copied); } + + public String codegenKey() { + List, JsonCodec>> entries = new ArrayList<>(codecs.entrySet()); + entries.sort(Comparator.comparing(entry -> entry.getKey().getName())); + StringBuilder builder = new StringBuilder(entries.size() * 48); + for (Map.Entry, JsonCodec> entry : entries) { + builder + .append(entry.getKey().getName()) + .append('=') + .append(entry.getValue().getClass().getName()) + .append(';'); + } + return builder.toString(); + } } diff --git a/java/fory-json/src/main/java/org/apache/fory/json/resolver/JsonSharedRegistry.java b/java/fory-json/src/main/java/org/apache/fory/json/resolver/JsonSharedRegistry.java index 4fe557959e..e8d2b95cf9 100644 --- a/java/fory-json/src/main/java/org/apache/fory/json/resolver/JsonSharedRegistry.java +++ b/java/fory-json/src/main/java/org/apache/fory/json/resolver/JsonSharedRegistry.java @@ -56,10 +56,14 @@ import java.util.UUID; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicIntegerArray; import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicLongArray; import java.util.concurrent.atomic.AtomicReference; +import java.util.concurrent.atomic.AtomicReferenceArray; import java.util.regex.Pattern; import org.apache.fory.json.ForyJsonException; +import org.apache.fory.json.JsonConfig; import org.apache.fory.json.codec.ArrayCodec; import org.apache.fory.json.codec.BaseObjectCodec; import org.apache.fory.json.codec.CodecUtils; @@ -79,12 +83,16 @@ public final class JsonSharedRegistry { private final CodecRegistry customCodecs; private final IdentityHashMap, JsonCodec> exactCodecs; private final JsonCodegen codegen; + private final boolean propertyDiscoveryEnabled; - public JsonSharedRegistry( - boolean codegenEnabled, boolean writeNullFields, CodecRegistry customCodecs) { - this.customCodecs = customCodecs.copy(); + public JsonSharedRegistry(JsonConfig config) { + this.customCodecs = config.codecRegistry().copy(); + this.propertyDiscoveryEnabled = config.propertyDiscoveryEnabled(); exactCodecs = new IdentityHashMap<>(); - codegen = codegenEnabled ? new JsonCodegen(writeNullFields) : null; + codegen = + config.codegenEnabled() + ? new JsonCodegen(config.writeNullFields(), config.getConfigHash()) + : null; registerExactCodecs(); } @@ -116,6 +124,10 @@ public JsonCodec createCodec( return new ScalarCodecs.AtomicReferenceCodec( CodecUtils.elementType(typeRef.getType()), localResolver); } + if (rawType == AtomicReferenceArray.class) { + return new ScalarCodecs.AtomicReferenceArrayCodec( + CodecUtils.elementType(typeRef.getType()), localResolver); + } if (Calendar.class.isAssignableFrom(rawType)) { return ScalarCodecs.CalendarCodec.INSTANCE; } @@ -190,6 +202,10 @@ public ObjectCodecs compileObject(BaseObjectCodec codec, JsonTypeResolver localR return codegen == null ? null : codegen.compile(codec, localResolver); } + boolean propertyDiscoveryEnabled() { + return propertyDiscoveryEnabled; + } + private void registerExactCodecs() { exactCodecs.put(Object.class, ScalarCodecs.NaturalCodec.INSTANCE); exactCodecs.put(void.class, ScalarCodecs.VoidCodec.INSTANCE); @@ -219,7 +235,9 @@ private void registerExactCodecs() { exactCodecs.put(StringBuffer.class, ScalarCodecs.StringBufferCodec.INSTANCE); exactCodecs.put(AtomicBoolean.class, ScalarCodecs.AtomicBooleanCodec.INSTANCE); exactCodecs.put(AtomicInteger.class, ScalarCodecs.AtomicIntegerCodec.INSTANCE); + exactCodecs.put(AtomicIntegerArray.class, ScalarCodecs.AtomicIntegerArrayCodec.INSTANCE); exactCodecs.put(AtomicLong.class, ScalarCodecs.AtomicLongCodec.INSTANCE); + exactCodecs.put(AtomicLongArray.class, ScalarCodecs.AtomicLongArrayCodec.INSTANCE); exactCodecs.put(Currency.class, ScalarCodecs.CurrencyCodec.INSTANCE); exactCodecs.put(File.class, ScalarCodecs.FileCodec.INSTANCE); exactCodecs.put(URI.class, ScalarCodecs.UriCodec.INSTANCE); diff --git a/java/fory-json/src/main/java/org/apache/fory/json/resolver/JsonTypeResolver.java b/java/fory-json/src/main/java/org/apache/fory/json/resolver/JsonTypeResolver.java index 0b23bbb38b..8b0e43e744 100644 --- a/java/fory-json/src/main/java/org/apache/fory/json/resolver/JsonTypeResolver.java +++ b/java/fory-json/src/main/java/org/apache/fory/json/resolver/JsonTypeResolver.java @@ -81,7 +81,7 @@ private BaseObjectCodec buildObjectCodec(Class type) { if (cached != null) { return cached; } - ObjectCodec codec = BaseObjectCodec.build(type); + ObjectCodec codec = BaseObjectCodec.build(type, sharedRegistry.propertyDiscoveryEnabled()); // Codegen may ask for nested object metadata that points back to this type. // Publishing before compiling keeps recursive ownership in this resolver cache. objectCodecs.put(type, codec); diff --git a/java/fory-json/src/main/java/org/apache/fory/json/writer/GeneratedObjectWriter.java b/java/fory-json/src/main/java/org/apache/fory/json/writer/GeneratedObjectWriter.java deleted file mode 100644 index 130ae08812..0000000000 --- a/java/fory-json/src/main/java/org/apache/fory/json/writer/GeneratedObjectWriter.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.fory.json.writer; - -import org.apache.fory.json.codec.JsonCodec; -import org.apache.fory.json.meta.JsonFieldInfo; - -/** Immutable metadata carrier shared by generated JSON object writers. */ -public abstract class GeneratedObjectWriter { - protected final JsonFieldInfo[] fields; - protected final JsonCodec[] codecs; - - protected GeneratedObjectWriter(JsonFieldInfo[] fields, JsonCodec[] codecs) { - this.fields = fields; - this.codecs = codecs; - } -} diff --git a/java/fory-json/src/main/java/org/apache/fory/json/writer/JdkDoubleFormatter.java b/java/fory-json/src/main/java/org/apache/fory/json/writer/JdkDoubleFormatter.java new file mode 100644 index 0000000000..0491264b2d --- /dev/null +++ b/java/fory-json/src/main/java/org/apache/fory/json/writer/JdkDoubleFormatter.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.json.writer; + +import java.lang.invoke.MethodHandle; +import java.lang.invoke.MethodHandles.Lookup; +import java.lang.invoke.MethodType; +import org.apache.fory.json.ForyJsonException; +import org.apache.fory.platform.AndroidSupport; +import org.apache.fory.platform.GraalvmSupport; +import org.apache.fory.platform.internal._JDKAccess; + +/** Exact JDK double formatter access for UTF-8 JSON number output. */ +final class JdkDoubleFormatter { + private static final MethodHandle PUT_DECIMAL = loadPutDecimal(); + + private JdkDoubleFormatter() {} + + static int write(byte[] buffer, int position, double value) { + MethodHandle putDecimal = PUT_DECIMAL; + if (putDecimal == null) { + return -1; + } + try { + return (int) putDecimal.invokeExact(buffer, position, value); + } catch (Throwable e) { + throw new ForyJsonException("Cannot write JSON double " + value, e); + } + } + + private static MethodHandle loadPutDecimal() { + if (AndroidSupport.IS_ANDROID || GraalvmSupport.IN_GRAALVM_NATIVE_IMAGE) { + return null; + } + try { + // This is the same formatter used by Double.toString; using it directly avoids materializing + // the intermediate String while preserving JDK numeric spelling exactly. + Class formatterClass = Class.forName("jdk.internal.math.DoubleToDecimal"); + Lookup lookup = _JDKAccess._trustedLookup(formatterClass); + Object latin1 = lookup.findStaticGetter(formatterClass, "LATIN1", formatterClass).invoke(); + MethodHandle putDecimal = + lookup.findVirtual( + formatterClass, + "putDecimal", + MethodType.methodType(int.class, byte[].class, int.class, double.class)); + return putDecimal + .bindTo(latin1) + .asType(MethodType.methodType(int.class, byte[].class, int.class, double.class)); + } catch (ClassNotFoundException | NoSuchFieldException | NoSuchMethodException e) { + return null; + } catch (IllegalAccessException e) { + return null; + } catch (Throwable e) { + throw new ExceptionInInitializerError(e); + } + } +} diff --git a/java/fory-json/src/main/java/org/apache/fory/json/writer/StringJsonWriter.java b/java/fory-json/src/main/java/org/apache/fory/json/writer/StringJsonWriter.java index ebe10dcffc..a726441e23 100644 --- a/java/fory-json/src/main/java/org/apache/fory/json/writer/StringJsonWriter.java +++ b/java/fory-json/src/main/java/org/apache/fory/json/writer/StringJsonWriter.java @@ -20,7 +20,9 @@ package org.apache.fory.json.writer; import java.nio.charset.StandardCharsets; +import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import org.apache.fory.json.ForyJsonException; import org.apache.fory.json.meta.JsonFieldInfo; import org.apache.fory.memory.LittleEndian; @@ -30,28 +32,39 @@ public final class StringJsonWriter extends JsonWriter { private static final byte LATIN1 = 0; private static final byte UTF16 = 1; - private static final int RETAINED_CAPACITY = 8192; + // Pooled writers should retain medium buffers to avoid reallocating common JSON outputs. + private static final int RETAINED_CAPACITY = 64 * 1024; private static final byte[] MIN_INT_BYTES = "-2147483648".getBytes(StandardCharsets.ISO_8859_1); private static final byte[] MIN_LONG_BYTES = "-9223372036854775808".getBytes(StandardCharsets.ISO_8859_1); + private static final long DECIMAL_8 = 100_000_000L; private static final long HIGH_BITS = 0x8080808080808080L; private static final int INT_HIGH_BITS = 0x80808080; private static final long ASCII_CONTROL_OFFSET = 0x6060606060606060L; private static final int INT_ASCII_CONTROL_OFFSET = 0x60606060; + private static final long ASCII_GT_QUOTE_OFFSET = 0x5D5D5D5D5D5D5D5DL; + private static final int INT_ASCII_GT_QUOTE_OFFSET = 0x5D5D5D5D; private static final long ONE_BYTES = 0x0101010101010101L; private static final int INT_ONE_BYTES = 0x01010101; private static final long QUOTE_BYTES_COMPLEMENT = ~0x2222222222222222L; private static final int INT_QUOTE_BYTES_COMPLEMENT = ~0x22222222; private static final long BACKSLASH_BYTES_COMPLEMENT = ~0x5C5C5C5C5C5C5C5CL; private static final int INT_BACKSLASH_BYTES_COMPLEMENT = ~0x5C5C5C5C; - private static final byte[] DIGIT_HUNDREDS = new byte[1000]; - private static final byte[] DIGIT_TENS = new byte[1000]; - private static final byte[] DIGIT_ONES = new byte[1000]; private static final int[] DIGIT_TRIPLES = new int[1000]; private static final int[] DIGIT_QUADS = new int[10000]; + private static final long[] UTF16_DIGIT_QUADS = new long[10000]; private static final long UTF16_BYTE_MASK = 0x00FF00FF00FF00FFL; private static final long UTF16_PAIR_MASK = 0x0000FFFF0000FFFFL; + private static final long UTF16_ONES = 0x0001000100010001L; + private static final long UTF16_HIGH_BITS = 0x8000800080008000L; + private static final long UTF16_QUOTE_CHARS = 0x0022002200220022L; + private static final long UTF16_BACKSLASH_CHARS = 0x005c005c005c005cL; + private static final long UTF16_CONTROL_LIMIT = 0x0020002000200020L; + private static final long UTF16_SURROGATE_MASK = 0xf800f800f800f800L; + private static final long UTF16_SURROGATE_PREFIX = 0xd800d800d800d800L; private static final boolean STRING_BYTES_BACKED = StringSerializer.isBytesBackedString(); + // Compact byte-backed Strings use one byte per char for LATIN1 and two bytes per char for UTF16. + // Length checks keep hot JSON string writing from loading the separate String coder field. private static final boolean LITTLE_ENDIAN = NativeByteOrder.IS_LITTLE_ENDIAN; static { @@ -59,9 +72,6 @@ public final class StringJsonWriter extends JsonWriter { int c0 = '0' + i / 100; int c1 = '0' + (i / 10) % 10; int c2 = '0' + i % 10; - DIGIT_HUNDREDS[i] = (byte) c0; - DIGIT_TENS[i] = (byte) c1; - DIGIT_ONES[i] = (byte) c2; int skip = i < 10 ? 2 : i < 100 ? 1 : 0; DIGIT_TRIPLES[i] = skip | (c0 << 8) | (c1 << 16) | (c2 << 24); } @@ -73,12 +83,16 @@ public final class StringJsonWriter extends JsonWriter { int c2 = '0' + low / 10; int c3 = '0' + low % 10; DIGIT_QUADS[i] = c0 | (c1 << 8) | (c2 << 16) | (c3 << 24); + long utf16 = spreadLatin1ToUtf16(DIGIT_QUADS[i] & 0xFFFFFFFFL); + UTF16_DIGIT_QUADS[i] = LITTLE_ENDIAN ? utf16 : utf16 << 8; } } private byte[] buffer; private byte[] scratch; private byte coder; + private byte nextCoder; + private boolean latin1Output; private int position; public StringJsonWriter(boolean writeNullFields) { @@ -98,15 +112,27 @@ public void reset() { if (scratch.length > RETAINED_CAPACITY) { scratch = new byte[RETAINED_CAPACITY]; } - coder = LATIN1; + coder = nextCoder; + latin1Output = coder == UTF16; position = 0; } public String toJson() { // The returned String may zero-copy this exact array, so pooled writer storage is never // exposed. - byte[] bytes = Arrays.copyOf(buffer, position); - return StringSerializer.newBytesStringZeroCopy(coder, bytes); + byte resultCoder = coder; + byte[] bytes; + if (coder == UTF16 && latin1Output) { + bytes = compressUtf16ToLatin1(buffer, position); + resultCoder = LATIN1; + } else { + bytes = Arrays.copyOf(buffer, position); + } + // Stable workloads often produce the same compact-string coder across operations. Remember the + // actual result coder so the next reset can avoid a LATIN1-to-UTF16 upgrade when UTF16 output + // repeats, while all-Latin1 outputs still materialize and continue as compact LATIN1 strings. + nextCoder = resultCoder; + return StringSerializer.newBytesStringZeroCopy(resultCoder, bytes); } @Override @@ -204,12 +230,17 @@ public void writeString(String value) { private void writeStringLatin1(String value) { if (STRING_BYTES_BACKED) { byte[] bytes = StringSerializer.getStringBytes(value); - byte stringCoder = StringSerializer.getStringCoder(value); - if (StringSerializer.isLatin1Coder(stringCoder)) { + int length = value.length(); + if (bytes.length == length) { ensure(bytes.length + 2); writeLatin1StringNoEnsure(bytes); return; } + if (LITTLE_ENDIAN) { + upgradeToUtf16((position << 1) + ((length + 2) << 1)); + writeUtf16StringBytes(value, bytes); + return; + } } ensure(value.length() * 6 + 2); writeStringCharsNoEnsure(value); @@ -269,6 +300,20 @@ public void writeIntField(byte[] namePrefix, byte[] commaNamePrefix, int index, writeIntField(prefix, value); } + public void writeIntField( + byte[] namePrefix, + byte[] commaNamePrefix, + byte[] utf16NamePrefix, + byte[] utf16CommaNamePrefix, + int index, + int value) { + if (coder == LATIN1) { + writeIntField(index == 0 ? namePrefix : commaNamePrefix, value); + return; + } + writeIntFieldUtf16Value(index == 0 ? utf16NamePrefix : utf16CommaNamePrefix, value); + } + public void writeIntField(byte[] prefix, int value) { if (coder == LATIN1) { writeIntFieldLatin1(prefix, value); @@ -277,6 +322,30 @@ public void writeIntField(byte[] prefix, int value) { writeIntFieldUtf16(prefix, value); } + public void writeIntField(byte[] prefix, byte[] utf16Prefix, int value) { + if (coder == LATIN1) { + writeIntFieldLatin1(prefix, value); + return; + } + writeIntFieldUtf16Value(utf16Prefix, value); + } + + public void writeIntField( + byte[] prefix, + long utf16Prefix0, + long utf16Prefix1, + long utf16Prefix2, + long utf16Prefix3, + int utf16PrefixLength, + int value) { + if (coder == LATIN1) { + writeIntFieldLatin1(prefix, value); + return; + } + writeIntFieldUtf16Packed( + utf16Prefix0, utf16Prefix1, utf16Prefix2, utf16Prefix3, utf16PrefixLength, value); + } + private void writeIntFieldLatin1(byte[] prefix, int value) { ensure(prefix.length + 11); writeRawLatin1NoEnsure(prefix); @@ -289,6 +358,25 @@ private void writeIntFieldUtf16(byte[] prefix, int value) { writeIntUtf16NoEnsure(value); } + private void writeIntFieldUtf16Value(byte[] utf16Prefix, int value) { + ensure(utf16Prefix.length + 22); + writeRawUtf16ValueNoEnsure(utf16Prefix); + writeIntUtf16NoEnsure(value); + } + + private void writeIntFieldUtf16Packed( + long utf16Prefix0, + long utf16Prefix1, + long utf16Prefix2, + long utf16Prefix3, + int utf16PrefixLength, + int value) { + ensurePackedUtf16Prefix(utf16PrefixLength, 22); + writePackedUtf16ValueNoEnsure( + utf16Prefix0, utf16Prefix1, utf16Prefix2, utf16Prefix3, utf16PrefixLength); + writeIntUtf16NoEnsure(value); + } + public void writeObjectIntField(byte[] namePrefix, int value) { if (coder == LATIN1) { writeObjectIntFieldLatin1(namePrefix, value); @@ -297,6 +385,30 @@ public void writeObjectIntField(byte[] namePrefix, int value) { writeObjectIntFieldUtf16(namePrefix, value); } + public void writeObjectIntField(byte[] namePrefix, byte[] utf16NamePrefix, int value) { + if (coder == LATIN1) { + writeObjectIntFieldLatin1(namePrefix, value); + return; + } + writeObjectIntFieldUtf16Value(utf16NamePrefix, value); + } + + public void writeObjectIntField( + byte[] namePrefix, + long utf16Prefix0, + long utf16Prefix1, + long utf16Prefix2, + long utf16Prefix3, + int utf16PrefixLength, + int value) { + if (coder == LATIN1) { + writeObjectIntFieldLatin1(namePrefix, value); + return; + } + writeObjectIntFieldUtf16Packed( + utf16Prefix0, utf16Prefix1, utf16Prefix2, utf16Prefix3, utf16PrefixLength, value); + } + private void writeObjectIntFieldLatin1(byte[] namePrefix, int value) { ensure(namePrefix.length + 12); buffer[position++] = (byte) '{'; @@ -311,11 +423,46 @@ private void writeObjectIntFieldUtf16(byte[] namePrefix, int value) { writeIntUtf16NoEnsure(value); } + private void writeObjectIntFieldUtf16Value(byte[] utf16NamePrefix, int value) { + ensure(utf16NamePrefix.length + 24); + writeUtf16ByteNoEnsure((byte) '{'); + writeRawUtf16ValueNoEnsure(utf16NamePrefix); + writeIntUtf16NoEnsure(value); + } + + private void writeObjectIntFieldUtf16Packed( + long utf16Prefix0, + long utf16Prefix1, + long utf16Prefix2, + long utf16Prefix3, + int utf16PrefixLength, + int value) { + ensurePackedUtf16Prefix(utf16PrefixLength, 24); + writeUtf16ByteNoEnsure((byte) '{'); + writePackedUtf16ValueNoEnsure( + utf16Prefix0, utf16Prefix1, utf16Prefix2, utf16Prefix3, utf16PrefixLength); + writeIntUtf16NoEnsure(value); + } + public void writeLongField(byte[] namePrefix, byte[] commaNamePrefix, int index, long value) { byte[] prefix = index == 0 ? namePrefix : commaNamePrefix; writeLongField(prefix, value); } + public void writeLongField( + byte[] namePrefix, + byte[] commaNamePrefix, + byte[] utf16NamePrefix, + byte[] utf16CommaNamePrefix, + int index, + long value) { + if (coder == LATIN1) { + writeLongField(index == 0 ? namePrefix : commaNamePrefix, value); + return; + } + writeLongFieldUtf16Value(index == 0 ? utf16NamePrefix : utf16CommaNamePrefix, value); + } + public void writeLongField(byte[] prefix, long value) { if (coder == LATIN1) { writeLongFieldLatin1(prefix, value); @@ -324,6 +471,30 @@ public void writeLongField(byte[] prefix, long value) { writeLongFieldUtf16(prefix, value); } + public void writeLongField(byte[] prefix, byte[] utf16Prefix, long value) { + if (coder == LATIN1) { + writeLongFieldLatin1(prefix, value); + return; + } + writeLongFieldUtf16Value(utf16Prefix, value); + } + + public void writeLongField( + byte[] prefix, + long utf16Prefix0, + long utf16Prefix1, + long utf16Prefix2, + long utf16Prefix3, + int utf16PrefixLength, + long value) { + if (coder == LATIN1) { + writeLongFieldLatin1(prefix, value); + return; + } + writeLongFieldUtf16Packed( + utf16Prefix0, utf16Prefix1, utf16Prefix2, utf16Prefix3, utf16PrefixLength, value); + } + private void writeLongFieldLatin1(byte[] prefix, long value) { ensure(prefix.length + 20); writeRawLatin1NoEnsure(prefix); @@ -336,6 +507,25 @@ private void writeLongFieldUtf16(byte[] prefix, long value) { writeLongUtf16NoEnsure(value); } + private void writeLongFieldUtf16Value(byte[] utf16Prefix, long value) { + ensure(utf16Prefix.length + 40); + writeRawUtf16ValueNoEnsure(utf16Prefix); + writeLongUtf16NoEnsure(value); + } + + private void writeLongFieldUtf16Packed( + long utf16Prefix0, + long utf16Prefix1, + long utf16Prefix2, + long utf16Prefix3, + int utf16PrefixLength, + long value) { + ensurePackedUtf16Prefix(utf16PrefixLength, 40); + writePackedUtf16ValueNoEnsure( + utf16Prefix0, utf16Prefix1, utf16Prefix2, utf16Prefix3, utf16PrefixLength); + writeLongUtf16NoEnsure(value); + } + public void writeObjectLongField(byte[] namePrefix, long value) { if (coder == LATIN1) { writeObjectLongFieldLatin1(namePrefix, value); @@ -344,6 +534,30 @@ public void writeObjectLongField(byte[] namePrefix, long value) { writeObjectLongFieldUtf16(namePrefix, value); } + public void writeObjectLongField(byte[] namePrefix, byte[] utf16NamePrefix, long value) { + if (coder == LATIN1) { + writeObjectLongFieldLatin1(namePrefix, value); + return; + } + writeObjectLongFieldUtf16Value(utf16NamePrefix, value); + } + + public void writeObjectLongField( + byte[] namePrefix, + long utf16Prefix0, + long utf16Prefix1, + long utf16Prefix2, + long utf16Prefix3, + int utf16PrefixLength, + long value) { + if (coder == LATIN1) { + writeObjectLongFieldLatin1(namePrefix, value); + return; + } + writeObjectLongFieldUtf16Packed( + utf16Prefix0, utf16Prefix1, utf16Prefix2, utf16Prefix3, utf16PrefixLength, value); + } + private void writeObjectLongFieldLatin1(byte[] namePrefix, long value) { ensure(namePrefix.length + 21); buffer[position++] = (byte) '{'; @@ -358,11 +572,46 @@ private void writeObjectLongFieldUtf16(byte[] namePrefix, long value) { writeLongUtf16NoEnsure(value); } + private void writeObjectLongFieldUtf16Value(byte[] utf16NamePrefix, long value) { + ensure(utf16NamePrefix.length + 42); + writeUtf16ByteNoEnsure((byte) '{'); + writeRawUtf16ValueNoEnsure(utf16NamePrefix); + writeLongUtf16NoEnsure(value); + } + + private void writeObjectLongFieldUtf16Packed( + long utf16Prefix0, + long utf16Prefix1, + long utf16Prefix2, + long utf16Prefix3, + int utf16PrefixLength, + long value) { + ensurePackedUtf16Prefix(utf16PrefixLength, 42); + writeUtf16ByteNoEnsure((byte) '{'); + writePackedUtf16ValueNoEnsure( + utf16Prefix0, utf16Prefix1, utf16Prefix2, utf16Prefix3, utf16PrefixLength); + writeLongUtf16NoEnsure(value); + } + public void writeStringField(byte[] namePrefix, byte[] commaNamePrefix, int index, String value) { byte[] prefix = index == 0 ? namePrefix : commaNamePrefix; writeStringField(prefix, value); } + public void writeStringField( + byte[] namePrefix, + byte[] commaNamePrefix, + byte[] utf16NamePrefix, + byte[] utf16CommaNamePrefix, + int index, + String value) { + if (coder == LATIN1) { + writeStringField(index == 0 ? namePrefix : commaNamePrefix, value); + return; + } + writeStringFieldUtf16Value(index == 0 ? utf16NamePrefix : utf16CommaNamePrefix, value); + } + public void writeStringField(byte[] prefix, String value) { if (coder == LATIN1) { writeStringFieldLatin1(prefix, value); @@ -371,16 +620,58 @@ public void writeStringField(byte[] prefix, String value) { writeStringFieldUtf16(prefix, value); } + public void writeStringField(byte[] prefix, byte[] utf16Prefix, String value) { + if (coder == LATIN1) { + writeStringFieldLatin1(prefix, utf16Prefix, value); + return; + } + writeStringFieldUtf16Value(utf16Prefix, value); + } + + public void writeStringField( + byte[] prefix, + long utf16Prefix0, + long utf16Prefix1, + long utf16Prefix2, + long utf16Prefix3, + int utf16PrefixLength, + String value) { + if (coder == LATIN1) { + writeStringFieldLatin1(prefix, null, value); + return; + } + writeStringFieldUtf16Packed( + utf16Prefix0, utf16Prefix1, utf16Prefix2, utf16Prefix3, utf16PrefixLength, value); + } + private void writeStringFieldLatin1(byte[] prefix, String value) { + writeStringFieldLatin1(prefix, null, value); + } + + private void writeStringFieldLatin1(byte[] prefix, byte[] utf16Prefix, String value) { if (STRING_BYTES_BACKED) { byte[] bytes = StringSerializer.getStringBytes(value); - byte stringCoder = StringSerializer.getStringCoder(value); - if (StringSerializer.isLatin1Coder(stringCoder)) { + int length = value.length(); + if (bytes.length == length) { ensure(prefix.length + bytes.length + 2); writeRawLatin1NoEnsure(prefix); writeLatin1StringNoEnsure(bytes); return; } + if (LITTLE_ENDIAN) { + int utf16PrefixLength = utf16Prefix == null ? prefix.length << 1 : utf16Prefix.length; + // A compact UTF16 input cannot remain in a Latin1 JSON String. Upgrade before writing the + // pending prefix so it is emitted once in the final coder instead of being written as + // Latin1 and widened immediately by the first non-Latin1 value character. + upgradeToUtf16((position << 1) + utf16PrefixLength + ((length + 2) << 1)); + if (utf16Prefix == null) { + writeRawUtf16NoEnsure(prefix); + } else { + writeRawUtf16ValueNoEnsure(utf16Prefix); + } + writeUtf16StringBytes(value, bytes); + return; + } } ensure(prefix.length + value.length() * 6 + 2); writeRawLatin1NoEnsure(prefix); @@ -393,6 +684,25 @@ private void writeStringFieldUtf16(byte[] prefix, String value) { writeString(value); } + private void writeStringFieldUtf16Value(byte[] utf16Prefix, String value) { + ensure(utf16Prefix.length); + writeRawUtf16ValueNoEnsure(utf16Prefix); + writeStringUtf16(value); + } + + private void writeStringFieldUtf16Packed( + long utf16Prefix0, + long utf16Prefix1, + long utf16Prefix2, + long utf16Prefix3, + int utf16PrefixLength, + String value) { + ensurePackedUtf16Prefix(utf16PrefixLength, 0); + writePackedUtf16ValueNoEnsure( + utf16Prefix0, utf16Prefix1, utf16Prefix2, utf16Prefix3, utf16PrefixLength); + writeStringUtf16(value); + } + public void writeStringElement(int index, String value) { int comma = index == 0 ? 0 : 1; if (value == null) { @@ -408,11 +718,7 @@ public void writeStringElement(int index, String value) { private void writeNullStringElement(int comma) { if (coder == UTF16) { - ensure((comma + 4) << 1); - if (comma != 0) { - writeUtf16ByteNoEnsure((byte) ','); - } - writeAsciiUtf16NoEnsure("null", 4); + writeNullStringElementUtf16(comma); return; } ensure(comma + 4); @@ -422,10 +728,19 @@ private void writeNullStringElement(int comma) { writeAsciiLatin1NoEnsure("null"); } + private void writeNullStringElementUtf16(int comma) { + ensure((comma + 4) << 1); + if (comma != 0) { + writeUtf16ByteNoEnsure((byte) ','); + } + writeAsciiUtf16NoEnsure("null", 4); + } + private void writeStringElementLatin1(int comma, String value) { if (STRING_BYTES_BACKED) { byte[] bytes = StringSerializer.getStringBytes(value); - if (StringSerializer.isLatin1Coder(StringSerializer.getStringCoder(value))) { + int length = value.length(); + if (bytes.length == length) { ensure(comma + bytes.length + 2); if (comma != 0) { buffer[position++] = ','; @@ -433,6 +748,14 @@ private void writeStringElementLatin1(int comma, String value) { writeLatin1StringNoEnsure(bytes); return; } + if (LITTLE_ENDIAN) { + upgradeToUtf16((position << 1) + ((comma + length + 2) << 1)); + if (comma != 0) { + writeUtf16ByteNoEnsure((byte) ','); + } + writeUtf16StringBytes(value, bytes); + return; + } } ensure(comma + value.length() * 6 + 2); if (comma != 0) { @@ -449,10 +772,96 @@ private void writeStringElementUtf16(int comma, String value) { writeStringUtf16(value); } + private void writeStringElementUtf16Nullable(int comma, String value) { + if (value == null) { + writeNullStringElementUtf16(comma); + return; + } + writeStringElementUtf16(comma, value); + } + + public void writeStringCollection(Collection values) { + writeArrayStart(); + if (coder == UTF16) { + writeStringCollectionUtf16(values); + writeArrayEnd(); + return; + } + if (values.getClass() == ArrayList.class) { + ArrayList list = (ArrayList) values; + int size = list.size(); + if (size != 0) { + writeStringElement(0, list.get(0)); + for (int i = 1; i < size; i++) { + writeStringElement(1, list.get(i)); + } + } + } else { + int index = 0; + for (String value : values) { + writeStringElement(index++, value); + } + } + writeArrayEnd(); + } + + private void writeStringCollectionUtf16(Collection values) { + if (values.getClass() == ArrayList.class) { + ArrayList list = (ArrayList) values; + int size = list.size(); + if (size != 0) { + writeStringElementUtf16Nullable(0, list.get(0)); + for (int i = 1; i < size; i++) { + writeStringElementUtf16Nullable(1, list.get(i)); + } + } + } else { + int index = 0; + for (String value : values) { + writeStringElementUtf16Nullable(index++, value); + } + } + } + public void writeRawValue(byte[] value) { writeRaw(value); } + public void writeRawValue(byte[] value, byte[] utf16Value) { + if (coder == LATIN1) { + writeRaw(value); + return; + } + writeRawUtf16Value(utf16Value); + } + + public void writeRawValue( + byte[] value, + long utf16Value0, + long utf16Value1, + long utf16Value2, + long utf16Value3, + int utf16Length) { + if (coder == LATIN1) { + writeRaw(value); + return; + } + writePackedUtf16Value(utf16Value0, utf16Value1, utf16Value2, utf16Value3, utf16Length); + } + + public void writeRawValue( + byte[] namePrefix, + byte[] commaNamePrefix, + byte[] utf16NamePrefix, + byte[] utf16CommaNamePrefix, + int index) { + if (coder == LATIN1) { + writeRaw(index == 0 ? namePrefix : commaNamePrefix); + return; + } + writeRawUtf16Value(index == 0 ? utf16NamePrefix : utf16CommaNamePrefix); + } + @Override public void writeObjectStart() { writeByteRaw((byte) '{'); @@ -644,12 +1053,31 @@ private void writeEscapedChar(char ch) { } private void writeStringUtf16(String value) { + if (STRING_BYTES_BACKED) { + byte[] bytes = StringSerializer.getStringBytes(value); + int length = value.length(); + if (bytes.length == length) { + writeLatin1StringUtf16(bytes); + return; + } + if (LITTLE_ENDIAN) { + writeUtf16StringBytes(value, bytes); + return; + } + } + writeStringUtf16Chars(value); + } + + private void writeStringUtf16Chars(String value) { int length = value.length(); ensure((length + 2) << 1); writeUtf16ByteNoEnsure((byte) '"'); for (int i = 0; i < length; i++) { char ch = value.charAt(i); if (isJsonUtf16(ch)) { + if (ch > 0xff) { + latin1Output = false; + } writeUtf16CharNoEnsure(ch); } else { writeStringUtf16Slow(value, i, length); @@ -659,6 +1087,95 @@ private void writeStringUtf16(String value) { writeByteRaw((byte) '"'); } + private void writeUtf16StringBytes(String value, byte[] source) { + latin1Output = false; + int length = value.length(); + int byteLength = length << 1; + ensure(byteLength + 4); + writeUtf16ByteNoEnsure((byte) '"'); + byte[] target = buffer; + int pos = position; + int index = 0; + int wordEnd = byteLength & ~7; + for (; index < wordEnd; index += 8) { + long word = LittleEndian.getInt64(source, index); + long quote = word ^ UTF16_QUOTE_CHARS; + long backslash = word ^ UTF16_BACKSLASH_CHARS; + long surrogate = (word & UTF16_SURROGATE_MASK) ^ UTF16_SURROGATE_PREFIX; + long stop = + ((quote - UTF16_ONES) & ~quote & UTF16_HIGH_BITS) + | ((backslash - UTF16_ONES) & ~backslash & UTF16_HIGH_BITS) + | ((word - UTF16_CONTROL_LIMIT) & ~word & UTF16_HIGH_BITS) + | ((surrogate - UTF16_ONES) & ~surrogate & UTF16_HIGH_BITS); + if (stop != 0) { + position = pos; + writeStringUtf16Slow(value, index >>> 1, length); + return; + } + LittleEndian.putInt64(target, pos, word); + pos += 8; + } + for (; index < byteLength; index += 2) { + char ch = (char) ((source[index] & 0xff) | ((source[index + 1] & 0xff) << 8)); + if (!isJsonUtf16(ch)) { + position = pos; + writeStringUtf16Slow(value, index >>> 1, length); + return; + } + target[pos] = source[index]; + target[pos + 1] = source[index + 1]; + pos += 2; + } + position = pos; + writeUtf16ByteNoEnsure((byte) '"'); + } + + private void writeLatin1StringUtf16(byte[] value) { + int length = value.length; + ensure((length + 2) << 1); + writeUtf16ByteNoEnsure((byte) '"'); + byte[] target = buffer; + int pos = position; + int i = 0; + int upperBound = length & ~7; + for (; i < upperBound; i += 8) { + long word = LittleEndian.getInt64(value, i); + if (!isJsonAsciiWord(word)) { + break; + } + putLatin1WordAsUtf16(target, pos, word); + pos += 16; + } + if (i + 4 <= length) { + int word = LittleEndian.getInt32(value, i); + if (isJsonAsciiInt(word)) { + putLatin1IntAsUtf16(target, pos, word); + pos += 8; + i += 4; + } + } + while (i < length) { + byte ch = value[i]; + if (isJsonLatin1Byte(ch)) { + pos = putUtf16Char(target, pos, (char) (ch & 0xff)); + i++; + } else { + position = pos; + writeLatin1StringUtf16Slow(value, i, length); + return; + } + } + position = pos; + writeUtf16ByteNoEnsure((byte) '"'); + } + + private void writeLatin1StringUtf16Slow(byte[] value, int index, int length) { + for (int i = index; i < length; i++) { + writeEscapedChar((char) (value[i] & 0xff)); + } + writeByteRaw((byte) '"'); + } + private void writeStringUtf16Slow(String value, int index, int length) { for (int i = index; i < length; i++) { char ch = value.charAt(i); @@ -670,6 +1187,7 @@ private void writeStringUtf16Slow(String value, int index, int length) { if (!Character.isLowSurrogate(low)) { throw new ForyJsonException("Unpaired high surrogate in string"); } + latin1Output = false; ensure(4); writeUtf16CharNoEnsure(ch); writeUtf16CharNoEnsure(low); @@ -767,6 +1285,39 @@ private void writeRawUtf16(byte[] bytes) { writeRawUtf16NoEnsure(bytes); } + private void writeRawUtf16Value(byte[] bytes) { + ensure(bytes.length); + writeRawUtf16ValueNoEnsure(bytes); + } + + private void writePackedUtf16Value( + long value0, long value1, long value2, long value3, int length) { + ensure(packedUtf16PrefixSize(length)); + writePackedUtf16ValueNoEnsure(value0, value1, value2, value3, length); + } + + private void writeRawUtf16ValueNoEnsure(byte[] bytes) { + System.arraycopy(bytes, 0, buffer, position, bytes.length); + position += bytes.length; + } + + private void writePackedUtf16ValueNoEnsure( + long value0, long value1, long value2, long value3, int length) { + byte[] target = buffer; + int pos = position; + LittleEndian.putInt64(target, pos, value0); + if (length > Long.BYTES) { + LittleEndian.putInt64(target, pos + Long.BYTES, value1); + if (length > Long.BYTES * 2) { + LittleEndian.putInt64(target, pos + Long.BYTES * 2, value2); + if (length > Long.BYTES * 3) { + LittleEndian.putInt64(target, pos + Long.BYTES * 3, value3); + } + } + } + position = pos + length; + } + private void writeRawUtf16NoEnsure(byte[] bytes) { byte[] target = buffer; int pos = position; @@ -805,6 +1356,9 @@ private void writeCharRaw(char value) { } private void writeCharRawUtf16(char value) { + if (value > 0xff) { + latin1Output = false; + } if (coder == LATIN1) { upgradeToUtf16((position << 1) + 2); } else { @@ -821,19 +1375,6 @@ private void reverse(int start, int end) { } } - private void reverseUtf16Chars(int start, int end) { - while (start < end) { - byte b0 = buffer[start]; - byte b1 = buffer[start + 1]; - buffer[start] = buffer[end]; - buffer[start + 1] = buffer[end + 1]; - buffer[end] = b0; - buffer[end + 1] = b1; - start += 2; - end -= 2; - } - } - private void writeUtf16ByteNoEnsure(byte value) { position = putUtf16Char(buffer, position, (char) (value & 0xff)); } @@ -860,6 +1401,14 @@ private void ensure(int additional) { } } + private void ensurePackedUtf16Prefix(int prefixLength, int additionalAfterPrefix) { + ensure(Math.max(packedUtf16PrefixSize(prefixLength), prefixLength + additionalAfterPrefix)); + } + + private static int packedUtf16PrefixSize(int prefixLength) { + return (prefixLength + Long.BYTES - 1) & -Long.BYTES; + } + private void grow(int minCapacity) { buffer = Arrays.copyOf(buffer, growCapacity(buffer.length, minCapacity)); } @@ -932,6 +1481,36 @@ private static long spreadLatin1ToUtf16(long value) { return (value | (value << 8)) & UTF16_BYTE_MASK; } + private static byte[] compressUtf16ToLatin1(byte[] source, int length) { + int latin1Length = length >>> 1; + byte[] target = new byte[latin1Length]; + if (LITTLE_ENDIAN) { + for (int i = 0, j = 0; j < latin1Length; i += 2, j++) { + target[j] = source[i]; + } + } else { + for (int i = 1, j = 0; j < latin1Length; i += 2, j++) { + target[j] = source[i]; + } + } + return target; + } + + private static void putLatin1WordAsUtf16(byte[] target, int offset, long word) { + if (LITTLE_ENDIAN) { + LittleEndian.putInt64(target, offset, spreadLatin1ToUtf16(word & 0xFFFFFFFFL)); + LittleEndian.putInt64(target, offset + 8, spreadLatin1ToUtf16(word >>> 32)); + } else { + LittleEndian.putInt64(target, offset, spreadLatin1ToUtf16(word & 0xFFFFFFFFL) << 8); + LittleEndian.putInt64(target, offset + 8, spreadLatin1ToUtf16(word >>> 32) << 8); + } + } + + private static void putLatin1IntAsUtf16(byte[] target, int offset, int word) { + long value = spreadLatin1ToUtf16(word & 0xFFFFFFFFL); + LittleEndian.putInt64(target, offset, LITTLE_ENDIAN ? value : value << 8); + } + private static boolean isJsonLatin1(char ch) { return ch > 0x1F && ch <= 0xff && ch != '"' && ch != '\\'; } @@ -946,16 +1525,23 @@ private static boolean isJsonLatin1Byte(byte value) { } private static boolean isJsonAsciiWord(long word) { + long notBackslash = ((word ^ BACKSLASH_BYTES_COMPLEMENT) + ONE_BYTES) & HIGH_BITS; + if ((notBackslash & (word + ASCII_GT_QUOTE_OFFSET)) == HIGH_BITS) { + return true; + } return (((word + ASCII_CONTROL_OFFSET) & ~word) & HIGH_BITS) == HIGH_BITS && (((word ^ QUOTE_BYTES_COMPLEMENT) + ONE_BYTES) & HIGH_BITS) == HIGH_BITS - && (((word ^ BACKSLASH_BYTES_COMPLEMENT) + ONE_BYTES) & HIGH_BITS) == HIGH_BITS; + && notBackslash == HIGH_BITS; } private static boolean isJsonAsciiInt(int word) { + int notBackslash = ((word ^ INT_BACKSLASH_BYTES_COMPLEMENT) + INT_ONE_BYTES) & INT_HIGH_BITS; + if ((notBackslash & (word + INT_ASCII_GT_QUOTE_OFFSET)) == INT_HIGH_BITS) { + return true; + } return (((word + INT_ASCII_CONTROL_OFFSET) & ~word) & INT_HIGH_BITS) == INT_HIGH_BITS && (((word ^ INT_QUOTE_BYTES_COMPLEMENT) + INT_ONE_BYTES) & INT_HIGH_BITS) == INT_HIGH_BITS - && (((word ^ INT_BACKSLASH_BYTES_COMPLEMENT) + INT_ONE_BYTES) & INT_HIGH_BITS) - == INT_HIGH_BITS; + && notBackslash == INT_HIGH_BITS; } private void writeIntNoEnsure(int value) { @@ -1017,11 +1603,13 @@ private void writeIntUtf16NoEnsure(int value) { writeRawUtf16NoEnsure(MIN_INT_BYTES); return; } + byte[] bytes = buffer; + int pos = position; if (value < 0) { - writeUtf16ByteNoEnsure((byte) '-'); + pos = putUtf16Byte(bytes, pos, (byte) '-'); value = -value; } - writePositiveIntUtf16NoEnsure(value); + position = writePositiveIntUtf16(bytes, pos, value); } private void writeLongUtf16NoEnsure(long value) { @@ -1029,20 +1617,13 @@ private void writeLongUtf16NoEnsure(long value) { writeRawUtf16NoEnsure(MIN_LONG_BYTES); return; } + byte[] bytes = buffer; + int pos = position; if (value < 0) { - writeUtf16ByteNoEnsure((byte) '-'); + pos = putUtf16Byte(bytes, pos, (byte) '-'); value = -value; } - if (value <= Integer.MAX_VALUE) { - writePositiveIntUtf16NoEnsure((int) value); - return; - } - int start = position; - do { - writeUtf16CharNoEnsure((char) ('0' + value % 10)); - value /= 10; - } while (value != 0); - reverseUtf16Chars(start, position - 2); + position = writePositiveLongUtf16(bytes, pos, value); } private void writePositiveIntNoEnsure(int value) { @@ -1069,23 +1650,40 @@ private void writePositiveIntNoEnsure(int value) { position = writePadded8(bytes, pos, middle, low); } - private void writePositiveIntUtf16NoEnsure(int value) { + private static int writePositiveIntUtf16(byte[] bytes, int pos, int value) { if (value < 10000) { - writeIntUpTo4Utf16(value); - return; + return writeIntUpTo4Utf16(bytes, pos, value); } int high = divide10000(value); int low = value - high * 10000; if (high < 10000) { - writeIntUpTo4Utf16(high); - writePadded4Utf16(low); - return; + if (high >= 1000) { + return writePadded8Utf16(bytes, pos, high, low); + } + pos = writeIntUpTo4Utf16(bytes, pos, high); + return writePadded4Utf16(bytes, pos, low); } int top = divide10000(high); int middle = high - top * 10000; - writeIntUpTo4Utf16(top); - writePadded4Utf16(middle); - writePadded4Utf16(low); + pos = writeIntUpTo4Utf16(bytes, pos, top); + return writePadded8Utf16(bytes, pos, middle, low); + } + + private static int writePositiveLongUtf16(byte[] bytes, int pos, long value) { + if (value <= Integer.MAX_VALUE) { + return writePositiveIntUtf16(bytes, pos, (int) value); + } + long high = value / DECIMAL_8; + int low = (int) (value - high * DECIMAL_8); + if (high < DECIMAL_8) { + pos = writePositiveIntUtf16(bytes, pos, (int) high); + return writePadded8Utf16(bytes, pos, low); + } + long top = high / DECIMAL_8; + int middle = (int) (high - top * DECIMAL_8); + pos = writePositiveIntUtf16(bytes, pos, (int) top); + pos = writePadded8Utf16(bytes, pos, middle); + return writePadded8Utf16(bytes, pos, low); } private static int divide10000(int value) { @@ -1117,32 +1715,69 @@ private static int writePadded8(byte[] bytes, int pos, int high, int low) { return pos + 8; } - private void writeIntUpTo4Utf16(int value) { - if (value < 10) { - writeUtf16ByteNoEnsure(DIGIT_ONES[value]); - } else if (value < 100) { - writeUtf16ByteNoEnsure(DIGIT_TENS[value]); - writeUtf16ByteNoEnsure(DIGIT_ONES[value]); - } else if (value < 1000) { - writePadded3Utf16(value); + private static int writeIntUpTo4Utf16(byte[] bytes, int pos, int value) { + if (value < 1000) { + return writeIntUpTo3Utf16(bytes, pos, value); + } + return writePadded4Utf16(bytes, pos, value); + } + + private static int writeIntUpTo3Utf16(byte[] bytes, int pos, int value) { + int digits = DIGIT_TRIPLES[value]; + int skip = digits & 0xFF; + int count = 3 - skip; + int chars = digits >>> ((skip + 1) << 3); + if (LITTLE_ENDIAN) { + bytes[pos] = (byte) chars; + bytes[pos + 1] = 0; + if (count > 1) { + bytes[pos + 2] = (byte) (chars >>> 8); + bytes[pos + 3] = 0; + if (count > 2) { + bytes[pos + 4] = (byte) (chars >>> 16); + bytes[pos + 5] = 0; + } + } } else { - writePadded4Utf16(value); + bytes[pos] = 0; + bytes[pos + 1] = (byte) chars; + if (count > 1) { + bytes[pos + 2] = 0; + bytes[pos + 3] = (byte) (chars >>> 8); + if (count > 2) { + bytes[pos + 4] = 0; + bytes[pos + 5] = (byte) (chars >>> 16); + } + } } + return pos + (count << 1); } - private void writePadded3Utf16(int value) { - writeUtf16ByteNoEnsure(DIGIT_HUNDREDS[value]); - writeUtf16ByteNoEnsure(DIGIT_TENS[value]); - writeUtf16ByteNoEnsure(DIGIT_ONES[value]); + private static int writePadded4Utf16(byte[] bytes, int pos, int value) { + LittleEndian.putInt64(bytes, pos, UTF16_DIGIT_QUADS[value]); + return pos + 8; } - private void writePadded4Utf16(int value) { - int high = value / 100; - int low = value - high * 100; - writeUtf16ByteNoEnsure((byte) ('0' + high / 10)); - writeUtf16ByteNoEnsure((byte) ('0' + high % 10)); - writeUtf16ByteNoEnsure((byte) ('0' + low / 10)); - writeUtf16ByteNoEnsure((byte) ('0' + low % 10)); + private static int writePadded8Utf16(byte[] bytes, int pos, int high, int low) { + pos = writePadded4Utf16(bytes, pos, high); + return writePadded4Utf16(bytes, pos, low); + } + + private static int writePadded8Utf16(byte[] bytes, int pos, int value) { + int high = divide10000(value); + int low = value - high * 10000; + return writePadded8Utf16(bytes, pos, high, low); + } + + private static int putUtf16Byte(byte[] bytes, int pos, byte value) { + if (LITTLE_ENDIAN) { + bytes[pos] = value; + bytes[pos + 1] = 0; + } else { + bytes[pos] = 0; + bytes[pos + 1] = value; + } + return pos + 2; } private static char hex(int value) { diff --git a/java/fory-json/src/main/java/org/apache/fory/json/writer/Utf8JsonWriter.java b/java/fory-json/src/main/java/org/apache/fory/json/writer/Utf8JsonWriter.java index 5ba1ec5e08..ad8502e6cd 100644 --- a/java/fory-json/src/main/java/org/apache/fory/json/writer/Utf8JsonWriter.java +++ b/java/fory-json/src/main/java/org/apache/fory/json/writer/Utf8JsonWriter.java @@ -19,28 +19,47 @@ package org.apache.fory.json.writer; +import java.io.IOException; +import java.io.OutputStream; +import java.time.LocalDate; +import java.time.OffsetDateTime; +import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; +import java.util.UUID; import org.apache.fory.json.ForyJsonException; import org.apache.fory.json.meta.JsonFieldInfo; import org.apache.fory.memory.LittleEndian; import org.apache.fory.serializer.StringSerializer; public final class Utf8JsonWriter extends JsonWriter { - private static final int RETAINED_CAPACITY = 8192; + // Pooled writers should retain medium buffers to avoid reallocating common JSON outputs. + private static final int RETAINED_CAPACITY = 64 * 1024; private static final byte[] MIN_INT_BYTES = "-2147483648".getBytes(java.nio.charset.StandardCharsets.ISO_8859_1); private static final byte[] MIN_LONG_BYTES = "-9223372036854775808".getBytes(java.nio.charset.StandardCharsets.ISO_8859_1); + private static final long EIGHT_DIGITS = 100_000_000L; private static final long HIGH_BITS = 0x8080808080808080L; private static final int INT_HIGH_BITS = 0x80808080; + private static final int SHORT_HIGH_BITS = 0x8080; private static final long ASCII_CONTROL_OFFSET = 0x6060606060606060L; private static final int INT_ASCII_CONTROL_OFFSET = 0x60606060; + private static final int SHORT_ASCII_CONTROL_OFFSET = 0x6060; + private static final long ASCII_GT_QUOTE_OFFSET = 0x5D5D5D5D5D5D5D5DL; + private static final int INT_ASCII_GT_QUOTE_OFFSET = 0x5D5D5D5D; + private static final int SHORT_ASCII_GT_QUOTE_OFFSET = 0x5D5D; private static final long ONE_BYTES = 0x0101010101010101L; private static final int INT_ONE_BYTES = 0x01010101; + private static final int SHORT_ONE_BYTES = 0x0101; + private static final byte[] HEX_DIGITS = + "0123456789abcdef".getBytes(java.nio.charset.StandardCharsets.ISO_8859_1); private static final long QUOTE_BYTES_COMPLEMENT = ~0x2222222222222222L; private static final int INT_QUOTE_BYTES_COMPLEMENT = ~0x22222222; + private static final int SHORT_QUOTE_BYTES_COMPLEMENT = ~0x2222; private static final long BACKSLASH_BYTES_COMPLEMENT = ~0x5C5C5C5C5C5C5C5CL; private static final int INT_BACKSLASH_BYTES_COMPLEMENT = ~0x5C5C5C5C; + private static final int SHORT_BACKSLASH_BYTES_COMPLEMENT = ~0x5C5C; private static final long UTF16_ASCII_MASK = 0xFF80FF80FF80FF80L; private static final int[] DIGIT_TRIPLES = new int[1000]; private static final int[] DIGIT_QUADS = new int[10000]; @@ -88,6 +107,14 @@ public byte[] toJsonBytes() { return Arrays.copyOf(buffer, position); } + public void writeTo(OutputStream output) { + try { + output.write(buffer, 0, position); + } catch (IOException e) { + throw new ForyJsonException("Cannot write JSON output", e); + } + } + @Override public void writeNull() { writeAscii("null"); @@ -115,16 +142,7 @@ public void writeLong(long value) { buffer[position++] = (byte) '-'; value = -value; } - if (value <= Integer.MAX_VALUE) { - writePositiveIntNoEnsure((int) value); - return; - } - int start = position; - do { - buffer[position++] = (byte) ('0' + value % 10); - value /= 10; - } while (value != 0); - reverse(start, position - 1); + writePositiveLongNoEnsure(value); } @Override @@ -132,7 +150,7 @@ public void writeFloat(float value) { if (!Float.isFinite(value)) { throw new ForyJsonException("JSON does not support non-finite float " + value); } - writeAscii(Float.toString(value)); + writeAsciiNumber(Float.toString(value)); } @Override @@ -140,12 +158,18 @@ public void writeDouble(double value) { if (!Double.isFinite(value)) { throw new ForyJsonException("JSON does not support non-finite double " + value); } - writeAscii(Double.toString(value)); + ensure(24); + int newPosition = JdkDoubleFormatter.write(buffer, position, value); + if (newPosition >= 0) { + position = newPosition; + return; + } + writeAsciiNumber(Double.toString(value)); } @Override public void writeNumber(String value) { - writeAscii(value); + writeAsciiNumber(value); } @Override @@ -162,12 +186,11 @@ public void writeChar(char value) { public void writeString(String value) { if (STRING_BYTES_BACKED) { byte[] bytes = StringSerializer.getStringBytes(value); - byte stringCoder = StringSerializer.getStringCoder(value); - if (StringSerializer.isLatin1Coder(stringCoder)) { + if (bytes.length == value.length()) { if (writeLatin1String(bytes)) { return; } - } else if (StringSerializer.isUtf16Coder(stringCoder)) { + } else { if (writeUtf16String(bytes)) { return; } @@ -176,6 +199,61 @@ public void writeString(String value) { writeStringChars(value); } + public void writeUuid(UUID value) { + ensure(38); + byte[] bytes = buffer; + int pos = position; + bytes[pos++] = (byte) '"'; + long high = value.getMostSignificantBits(); + pos = writeHex(bytes, pos, high, 60, 8); + bytes[pos++] = (byte) '-'; + pos = writeHex(bytes, pos, high, 28, 4); + bytes[pos++] = (byte) '-'; + pos = writeHex(bytes, pos, high, 12, 4); + long low = value.getLeastSignificantBits(); + bytes[pos++] = (byte) '-'; + pos = writeHex(bytes, pos, low, 60, 4); + bytes[pos++] = (byte) '-'; + pos = writeHex(bytes, pos, low, 44, 12); + bytes[pos++] = (byte) '"'; + position = pos; + } + + public void writeLocalDate(LocalDate value) { + int year = value.getYear(); + if (year < 0 || year > 9999) { + writeString(value.toString()); + return; + } + ensure(12); + byte[] bytes = buffer; + int pos = position; + bytes[pos++] = (byte) '"'; + pos = writeLocalDateBytes(bytes, pos, year, value.getMonthValue(), value.getDayOfMonth()); + bytes[pos++] = (byte) '"'; + position = pos; + } + + public void writeOffsetDateTime(OffsetDateTime value) { + int year = value.getYear(); + if (year < 0 || year > 9999 || value.getOffset().getTotalSeconds() != 0) { + writeString(value.toString()); + return; + } + ensure(32); + byte[] bytes = buffer; + int pos = position; + bytes[pos++] = (byte) '"'; + pos = writeLocalDateBytes(bytes, pos, year, value.getMonthValue(), value.getDayOfMonth()); + bytes[pos++] = (byte) 'T'; + pos = + writeTime( + bytes, pos, value.getHour(), value.getMinute(), value.getSecond(), value.getNano()); + bytes[pos++] = (byte) 'Z'; + bytes[pos++] = (byte) '"'; + position = pos; + } + private void writeStringChars(String value) { int length = value.length(); ensure(length + 2); @@ -258,12 +336,32 @@ public void writeIntField(byte[] namePrefix, byte[] commaNamePrefix, int index, writeIntField(prefix, value); } + public void writeIntField( + long namePrefix, + long commaPrefix, + int namePrefixLength, + int commaPrefixLength, + int index, + int value) { + if (index == 0) { + writeIntField(namePrefix, 0L, namePrefixLength, value); + } else { + writeIntField(commaPrefix, 0L, commaPrefixLength, value); + } + } + public void writeIntField(byte[] prefix, int value) { ensure(prefix.length + 11); writeRawNoEnsure(prefix); writeIntNoEnsure(value); } + public void writeIntField(long prefix0, long prefix1, int prefixLength, int value) { + ensurePackedPrefix(prefixLength, 11); + writePackedRawNoEnsure(prefix0, prefix1, prefixLength); + writeIntNoEnsure(value); + } + public void writeObjectIntField(byte[] namePrefix, int value) { ensure(namePrefix.length + 12); buffer[position++] = (byte) '{'; @@ -271,6 +369,13 @@ public void writeObjectIntField(byte[] namePrefix, int value) { writeIntNoEnsure(value); } + public void writeObjectIntField(long prefix0, long prefix1, int prefixLength, int value) { + ensurePackedPrefix(prefixLength, 12); + buffer[position++] = (byte) '{'; + writePackedRawNoEnsure(prefix0, prefix1, prefixLength); + writeIntNoEnsure(value); + } + public void writeLongField(byte[] namePrefix, byte[] commaNamePrefix, int index, long value) { byte[] prefix = index == 0 ? namePrefix : commaNamePrefix; writeLongField(prefix, value); @@ -282,6 +387,12 @@ public void writeLongField(byte[] prefix, long value) { writeLongNoEnsure(value); } + public void writeLongField(long prefix0, long prefix1, int prefixLength, long value) { + ensurePackedPrefix(prefixLength, 20); + writePackedRawNoEnsure(prefix0, prefix1, prefixLength); + writeLongNoEnsure(value); + } + public void writeObjectLongField(byte[] namePrefix, long value) { ensure(namePrefix.length + 21); buffer[position++] = (byte) '{'; @@ -289,24 +400,44 @@ public void writeObjectLongField(byte[] namePrefix, long value) { writeLongNoEnsure(value); } + public void writeObjectLongField(long prefix0, long prefix1, int prefixLength, long value) { + ensurePackedPrefix(prefixLength, 21); + buffer[position++] = (byte) '{'; + writePackedRawNoEnsure(prefix0, prefix1, prefixLength); + writeLongNoEnsure(value); + } + public void writeStringField(byte[] namePrefix, byte[] commaNamePrefix, int index, String value) { byte[] prefix = index == 0 ? namePrefix : commaNamePrefix; writeStringField(prefix, value); } + public void writeStringField( + long namePrefix, + long commaPrefix, + int namePrefixLength, + int commaPrefixLength, + int index, + String value) { + if (index == 0) { + writeStringField(namePrefix, 0L, namePrefixLength, value); + } else { + writeStringField(commaPrefix, 0L, commaPrefixLength, value); + } + } + public void writeStringField(byte[] prefix, String value) { if (STRING_BYTES_BACKED) { byte[] bytes = StringSerializer.getStringBytes(value); - byte stringCoder = StringSerializer.getStringCoder(value); int start = position; - if (StringSerializer.isLatin1Coder(stringCoder)) { + if (bytes.length == value.length()) { ensure(prefix.length + bytes.length + 2); writeRawNoEnsure(prefix); if (writeLatin1StringNoEnsure(bytes)) { return; } position = start; - } else if (StringSerializer.isUtf16Coder(stringCoder)) { + } else { ensure(prefix.length + (bytes.length >> 1) * 3 + 2); writeRawNoEnsure(prefix); if (writeUtf16StringNoEnsure(bytes)) { @@ -318,23 +449,117 @@ public void writeStringField(byte[] prefix, String value) { writeStringFieldChars(prefix, value); } + public void writeStringField(long prefix0, long prefix1, int prefixLength, String value) { + if (STRING_BYTES_BACKED) { + byte[] bytes = StringSerializer.getStringBytes(value); + int start = position; + if (bytes.length == value.length()) { + ensurePackedPrefix(prefixLength, bytes.length + 2); + writePackedRawNoEnsure(prefix0, prefix1, prefixLength); + if (writeLatin1StringNoEnsure(bytes)) { + return; + } + position = start; + } else { + ensurePackedPrefix(prefixLength, (bytes.length >> 1) * 3 + 2); + writePackedRawNoEnsure(prefix0, prefix1, prefixLength); + if (writeUtf16StringNoEnsure(bytes)) { + return; + } + position = start; + } + } + writeStringFieldChars(prefix0, prefix1, prefixLength, value); + } + private void writeStringFieldChars(byte[] prefix, String value) { ensure(prefix.length + value.length() * 3 + 2); writeRawNoEnsure(prefix); writeStringNoEnsure(value); } + private void writeStringFieldChars(long prefix0, long prefix1, int prefixLength, String value) { + ensurePackedPrefix(prefixLength, value.length() * 3 + 2); + writePackedRawNoEnsure(prefix0, prefix1, prefixLength); + writeStringNoEnsure(value); + } + + public void writeStringCollection(Collection values) { + writeArrayStart(); + if (values.getClass() == ArrayList.class) { + ArrayList list = (ArrayList) values; + int size = list.size(); + if (size != 0) { + writeStringElementWithComma(0, list.get(0)); + for (int i = 1; i < size; i++) { + writeStringElementWithComma(1, list.get(i)); + } + } + } else { + int index = 0; + for (String value : values) { + writeStringElementWithComma(index++ == 0 ? 0 : 1, value); + } + } + writeArrayEnd(); + } + + public void writeStringArray(String[] values) { + writeArrayStart(); + int length = values.length; + if (length != 0) { + int i = 1; + writeStringElementWithComma(0, values[0]); + if ((length & 1) == 0) { + writeStringElementWithComma(1, values[i]); + i++; + } + for (; i < length; i += 2) { + writeStringElementWithComma(1, values[i]); + writeStringElementWithComma(1, values[i + 1]); + } + } + writeArrayEnd(); + } + + public void writeLongArray(long[] values) { + ensure(2); + buffer[position++] = '['; + int length = values.length; + if (length != 0) { + ensure(22); + writeLongNoEnsure(values[0]); + int i = 1; + if ((length & 1) == 0) { + ensure(22); + buffer[position++] = ','; + writeLongNoEnsure(values[i]); + i++; + } + for (; i < length; i += 2) { + ensure(44); + buffer[position++] = ','; + writeLongNoEnsure(values[i]); + buffer[position++] = ','; + writeLongNoEnsure(values[i + 1]); + } + } + buffer[position++] = ']'; + } + public void writeStringElement(int index, String value) { - int comma = index == 0 ? 0 : 1; + writeStringElementWithComma(index == 0 ? 0 : 1, value); + } + + private void writeStringElementWithComma(int comma, String value) { if (value == null) { writeNullStringElement(comma); return; } if (STRING_BYTES_BACKED) { byte[] bytes = StringSerializer.getStringBytes(value); - byte stringCoder = StringSerializer.getStringCoder(value); int start = position; - if (StringSerializer.isLatin1Coder(stringCoder)) { + if (bytes.length == value.length()) { ensure(comma + bytes.length + 2); if (comma != 0) { buffer[position++] = ','; @@ -343,7 +568,7 @@ public void writeStringElement(int index, String value) { return; } position = start; - } else if (StringSerializer.isUtf16Coder(stringCoder)) { + } else { ensure(comma + (bytes.length >> 1) * 3 + 2); if (comma != 0) { buffer[position++] = ','; @@ -377,6 +602,11 @@ public void writeRawValue(byte[] value) { writeRaw(value); } + public void writeRawValue(long prefix0, long prefix1, int prefixLength) { + ensure(packedPrefixSize(prefixLength)); + writePackedRawNoEnsure(prefix0, prefix1, prefixLength); + } + @Override public void writeObjectStart() { writeByteRaw((byte) '{'); @@ -407,58 +637,249 @@ public void writeComma(int index) { private boolean writeLatin1String(byte[] value) { int length = value.length; ensure(length + 2); - return writeLatin1StringNoEnsure(value); + return writeLatin1StringNoEnsure(value, length); } private boolean writeLatin1StringNoEnsure(byte[] value) { int length = value.length; + return writeLatin1StringNoEnsure(value, length); + } + + private boolean writeLatin1StringNoEnsure(byte[] value, int length) { + if (length < 32) { + return writeShortLatin1StringNoEnsure(value, length); + } + return writeLongLatin1StringNoEnsure(value, length); + } + + private boolean writeLongLatin1StringNoEnsure(byte[] value, int length) { byte[] bytes = buffer; int start = position; + if (!isJsonAsciiBytes(value, length)) { + return false; + } int pos = start; bytes[pos++] = (byte) '"'; + System.arraycopy(value, 0, bytes, pos, length); + pos += length; + bytes[pos++] = (byte) '"'; + position = pos; + return true; + } + + private static boolean isJsonAsciiBytes(byte[] value, int length) { int i = 0; int upperBound = length & ~15; for (; i < upperBound; i += 16) { long word0 = LittleEndian.getInt64(value, i); long word1 = LittleEndian.getInt64(value, i + 8); if (!isJsonAsciiWord(word0) || !isJsonAsciiWord(word1)) { - break; + return false; } - LittleEndian.putInt64(bytes, pos, word0); - LittleEndian.putInt64(bytes, pos + 8, word1); - pos += 16; } upperBound = length & ~7; for (; i < upperBound; i += 8) { long word = LittleEndian.getInt64(value, i); if (!isJsonAsciiWord(word)) { - break; + return false; } - LittleEndian.putInt64(bytes, pos, word); - pos += 8; } if (i + 4 <= length) { int word = LittleEndian.getInt32(value, i); if (isJsonAsciiInt(word)) { - LittleEndian.putInt32(bytes, pos, word); - pos += 4; i += 4; + } else { + return false; } } - for (; i < length; i++) { - byte ch = value[i]; - if (isJsonAsciiByte(ch)) { - bytes[pos++] = ch; + if (i + 2 <= length) { + int word = (value[i] & 0xFF) | ((value[i + 1] & 0xFF) << 8); + if (isJsonAsciiShort(word)) { + i += 2; } else { + return false; + } + } + for (; i < length; i++) { + if (!isJsonAsciiByte(value[i])) { + return false; + } + } + return true; + } + + private boolean writeShortLatin1StringNoEnsure(byte[] value, int length) { + if (length > 24) { + return writeLatin1String25To31(value, length); + } + if (length > 16) { + return writeLatin1String17To24(value, length); + } + byte[] bytes = buffer; + int start = position; + int pos = start; + bytes[pos++] = (byte) '"'; + // Short compact strings dominate generated JSON writers. Keep the 8-16 byte path exact here; + // longer short-string bands stay in helpers so this common path remains small. + if (length >= 8) { + long word = LittleEndian.getInt64(value, 0); + if (!isJsonAsciiWord(word)) { position = start; return false; } + LittleEndian.putInt64(bytes, pos, word); + pos += 8; + int index = Long.BYTES; + if (index + Long.BYTES <= length) { + long tail = LittleEndian.getInt64(value, index); + if (!isJsonAsciiWord(tail)) { + position = start; + return false; + } + LittleEndian.putInt64(bytes, pos, tail); + pos += Long.BYTES; + index += Long.BYTES; + } + if (index + Integer.BYTES <= length) { + int tail = LittleEndian.getInt32(value, index); + if (!isJsonAsciiInt(tail)) { + position = start; + return false; + } + LittleEndian.putInt32(bytes, pos, tail); + pos += Integer.BYTES; + index += Integer.BYTES; + } + if (index + Short.BYTES <= length) { + int tail = (value[index] & 0xFF) | ((value[index + 1] & 0xFF) << 8); + if (!isJsonAsciiShort(tail)) { + position = start; + return false; + } + bytes[pos] = (byte) tail; + bytes[pos + 1] = (byte) (tail >>> 8); + pos += Short.BYTES; + index += Short.BYTES; + } + if (index < length) { + byte tail = value[index]; + if (!isJsonAsciiByte(tail)) { + position = start; + return false; + } + bytes[pos++] = tail; + } + } else { + // Keep the sub-8 tail outside this method so the common word-sized short-string path stays + // small enough to inline into generated object writers. + return writeLatin1String0To7(value, length); } bytes[pos++] = (byte) '"'; position = pos; return true; } + private boolean writeLatin1String0To7(byte[] value, int length) { + byte[] bytes = buffer; + int start = position; + int pos = start; + bytes[pos++] = (byte) '"'; + if (length >= 4) { + int word = LittleEndian.getInt32(value, 0); + if (!isJsonAsciiInt(word)) { + position = start; + return false; + } + LittleEndian.putInt32(bytes, pos, word); + if (length > 4) { + int tailOffset = length - Integer.BYTES; + int tail = LittleEndian.getInt32(value, tailOffset); + if (!isJsonAsciiInt(tail)) { + position = start; + return false; + } + LittleEndian.putInt32(bytes, pos + tailOffset, tail); + } + pos += length; + } else if (length >= 2) { + int word = (value[0] & 0xFF) | ((value[1] & 0xFF) << 8); + if (!isJsonAsciiShort(word)) { + position = start; + return false; + } + bytes[pos] = (byte) word; + bytes[pos + 1] = (byte) (word >>> 8); + if (length == 3) { + byte ch = value[2]; + if (!isJsonAsciiByte(ch)) { + position = start; + return false; + } + bytes[pos + 2] = ch; + } + pos += length; + } else if (length == 1) { + byte ch = value[0]; + if (!isJsonAsciiByte(ch)) { + position = start; + return false; + } + bytes[pos++] = ch; + } + bytes[pos++] = (byte) '"'; + position = pos; + return true; + } + + private boolean writeLatin1String25To31(byte[] value, int length) { + byte[] bytes = buffer; + int start = position; + int pos = start; + bytes[pos++] = (byte) '"'; + long word0 = LittleEndian.getInt64(value, 0); + long word1 = LittleEndian.getInt64(value, 8); + long word2 = LittleEndian.getInt64(value, 16); + int tailOffset = length - Long.BYTES; + long tail = LittleEndian.getInt64(value, tailOffset); + if (!isJsonAsciiWord(word0) + || !isJsonAsciiWord(word1) + || !isJsonAsciiWord(word2) + || !isJsonAsciiWord(tail)) { + position = start; + return false; + } + LittleEndian.putInt64(bytes, pos, word0); + LittleEndian.putInt64(bytes, pos + 8, word1); + LittleEndian.putInt64(bytes, pos + 16, word2); + LittleEndian.putInt64(bytes, pos + tailOffset, tail); + pos += length; + bytes[pos++] = (byte) '"'; + position = pos; + return true; + } + + private boolean writeLatin1String17To24(byte[] value, int length) { + byte[] bytes = buffer; + int start = position; + int pos = start; + bytes[pos++] = (byte) '"'; + long word0 = LittleEndian.getInt64(value, 0); + long word1 = LittleEndian.getInt64(value, 8); + int tailOffset = length - Long.BYTES; + long tail = LittleEndian.getInt64(value, tailOffset); + if (!isJsonAsciiWord(word0) || !isJsonAsciiWord(word1) || !isJsonAsciiWord(tail)) { + position = start; + return false; + } + LittleEndian.putInt64(bytes, pos, word0); + LittleEndian.putInt64(bytes, pos + 8, word1); + LittleEndian.putInt64(bytes, pos + tailOffset, tail); + pos += length; + bytes[pos++] = (byte) '"'; + position = pos; + return true; + } + private boolean writeUtf16String(byte[] value) { int length = value.length; ensure((length >> 1) * 3 + 2); @@ -485,8 +906,34 @@ private boolean writeUtf16StringNoEnsure(byte[] value) { LittleEndian.putInt32(bytes, pos, packed); pos += 4; } - for (; i < length; i += 2) { + while (i < length) { + if (i + 8 <= length) { + char c0 = StringSerializer.getBytesChar(value, i); + char c1 = StringSerializer.getBytesChar(value, i + 2); + char c2 = StringSerializer.getBytesChar(value, i + 4); + char c3 = StringSerializer.getBytesChar(value, i + 6); + if (isUtf8ThreeByte(c0) + && isUtf8ThreeByte(c1) + && isUtf8ThreeByte(c2) + && isUtf8ThreeByte(c3)) { + bytes[pos++] = (byte) (0xE0 | (c0 >>> 12)); + bytes[pos++] = (byte) (0x80 | ((c0 >>> 6) & 0x3F)); + bytes[pos++] = (byte) (0x80 | (c0 & 0x3F)); + bytes[pos++] = (byte) (0xE0 | (c1 >>> 12)); + bytes[pos++] = (byte) (0x80 | ((c1 >>> 6) & 0x3F)); + bytes[pos++] = (byte) (0x80 | (c1 & 0x3F)); + bytes[pos++] = (byte) (0xE0 | (c2 >>> 12)); + bytes[pos++] = (byte) (0x80 | ((c2 >>> 6) & 0x3F)); + bytes[pos++] = (byte) (0x80 | (c2 & 0x3F)); + bytes[pos++] = (byte) (0xE0 | (c3 >>> 12)); + bytes[pos++] = (byte) (0x80 | ((c3 >>> 6) & 0x3F)); + bytes[pos++] = (byte) (0x80 | (c3 & 0x3F)); + i += 8; + continue; + } + } char ch = StringSerializer.getBytesChar(value, i); + i += 2; if (ch < 0x80) { if (!isJsonAscii(ch)) { position = start; @@ -496,7 +943,7 @@ private boolean writeUtf16StringNoEnsure(byte[] value) { } else if (ch < 0x800) { bytes[pos++] = (byte) (0xC0 | (ch >>> 6)); bytes[pos++] = (byte) (0x80 | (ch & 0x3F)); - } else if (Character.isSurrogate(ch)) { + } else if (ch >= Character.MIN_SURROGATE && ch <= Character.MAX_SURROGATE) { position = start; return false; } else { @@ -510,6 +957,10 @@ private boolean writeUtf16StringNoEnsure(byte[] value) { return true; } + private static boolean isUtf8ThreeByte(char ch) { + return ch >= 0x800 && (ch < Character.MIN_SURROGATE || ch > Character.MAX_SURROGATE); + } + private void writeEscapedChar(char ch) { switch (ch) { case '"': @@ -575,12 +1026,11 @@ private void writeStringSlow(String value, int index, int length) { private void writeStringNoEnsure(String value) { if (STRING_BYTES_BACKED) { byte[] bytes = StringSerializer.getStringBytes(value); - byte stringCoder = StringSerializer.getStringCoder(value); - if (StringSerializer.isLatin1Coder(stringCoder)) { + if (bytes.length == value.length()) { if (writeLatin1StringNoEnsure(bytes)) { return; } - } else if (StringSerializer.isUtf16Coder(stringCoder)) { + } else { if (writeUtf16StringNoEnsure(bytes)) { return; } @@ -657,6 +1107,17 @@ private void writeAsciiNoEnsure(String value) { } } + private void writeAsciiNumber(String value) { + if (STRING_BYTES_BACKED) { + byte[] bytes = StringSerializer.getStringBytes(value); + if (bytes.length == value.length()) { + writeRaw(bytes); + return; + } + } + writeAscii(value); + } + private void writeRaw(byte[] bytes) { ensure(bytes.length); writeRawNoEnsure(bytes); @@ -667,19 +1128,27 @@ private void writeRawNoEnsure(byte[] bytes) { position += bytes.length; } + private void writePackedRawNoEnsure(long prefix0, long prefix1, int prefixLength) { + LittleEndian.putInt64(buffer, position, prefix0); + if (prefixLength > Long.BYTES) { + LittleEndian.putInt64(buffer, position + Long.BYTES, prefix1); + } + position += prefixLength; + } + + private void ensurePackedPrefix(int prefixLength, int additionalAfterPrefix) { + ensure(Math.max(packedPrefixSize(prefixLength), prefixLength + additionalAfterPrefix)); + } + + private static int packedPrefixSize(int prefixLength) { + return prefixLength <= Long.BYTES ? Long.BYTES : Long.BYTES * 2; + } + private void writeByteRaw(byte value) { ensure(1); buffer[position++] = value; } - private void reverse(int start, int end) { - while (start < end) { - byte tmp = buffer[start]; - buffer[start++] = buffer[end]; - buffer[end--] = tmp; - } - } - private void ensure(int additional) { int minCapacity = position + additional; if (minCapacity > buffer.length) { @@ -709,16 +1178,38 @@ private static boolean isJsonAsciiByte(byte value) { } private static boolean isJsonAsciiWord(long word) { + long notBackslashMask = ((word ^ BACKSLASH_BYTES_COMPLEMENT) + ONE_BYTES) & HIGH_BITS; + // Common unescaped bytes are greater than '"' and not '\\'. The fallback keeps the exact + // quote, backslash, control, and high-bit predicate for the remaining byte values. + if ((notBackslashMask & (word + ASCII_GT_QUOTE_OFFSET)) == HIGH_BITS) { + return true; + } return (((word + ASCII_CONTROL_OFFSET) & ~word) & HIGH_BITS) == HIGH_BITS && (((word ^ QUOTE_BYTES_COMPLEMENT) + ONE_BYTES) & HIGH_BITS) == HIGH_BITS - && (((word ^ BACKSLASH_BYTES_COMPLEMENT) + ONE_BYTES) & HIGH_BITS) == HIGH_BITS; + && notBackslashMask == HIGH_BITS; } private static boolean isJsonAsciiInt(int word) { + int notBackslashMask = + ((word ^ INT_BACKSLASH_BYTES_COMPLEMENT) + INT_ONE_BYTES) & INT_HIGH_BITS; + if ((notBackslashMask & (word + INT_ASCII_GT_QUOTE_OFFSET)) == INT_HIGH_BITS) { + return true; + } return (((word + INT_ASCII_CONTROL_OFFSET) & ~word) & INT_HIGH_BITS) == INT_HIGH_BITS && (((word ^ INT_QUOTE_BYTES_COMPLEMENT) + INT_ONE_BYTES) & INT_HIGH_BITS) == INT_HIGH_BITS - && (((word ^ INT_BACKSLASH_BYTES_COMPLEMENT) + INT_ONE_BYTES) & INT_HIGH_BITS) - == INT_HIGH_BITS; + && notBackslashMask == INT_HIGH_BITS; + } + + private static boolean isJsonAsciiShort(int word) { + int notBackslashMask = + ((word ^ SHORT_BACKSLASH_BYTES_COMPLEMENT) + SHORT_ONE_BYTES) & SHORT_HIGH_BITS; + if ((notBackslashMask & (word + SHORT_ASCII_GT_QUOTE_OFFSET)) == SHORT_HIGH_BITS) { + return true; + } + return (((word + SHORT_ASCII_CONTROL_OFFSET) & ~word) & SHORT_HIGH_BITS) == SHORT_HIGH_BITS + && (((word ^ SHORT_QUOTE_BYTES_COMPLEMENT) + SHORT_ONE_BYTES) & SHORT_HIGH_BITS) + == SHORT_HIGH_BITS + && notBackslashMask == SHORT_HIGH_BITS; } private static int packUtf16Ascii(long word) { @@ -750,40 +1241,124 @@ private void writeLongNoEnsure(long value) { buffer[position++] = (byte) '-'; value = -value; } + writePositiveLongNoEnsure(value); + } + + private void writePositiveIntNoEnsure(int value) { + position = writePositiveInt(buffer, position, value); + } + + private void writePositiveLongNoEnsure(long value) { if (value <= Integer.MAX_VALUE) { writePositiveIntNoEnsure((int) value); return; } - int start = position; - do { - buffer[position++] = (byte) ('0' + value % 10); - value /= 10; - } while (value != 0); - reverse(start, position - 1); - } - - private void writePositiveIntNoEnsure(int value) { byte[] bytes = buffer; int pos = position; + long high = value / EIGHT_DIGITS; + int low = (int) (value - high * EIGHT_DIGITS); + if (high <= Integer.MAX_VALUE) { + pos = writePositiveInt(bytes, pos, (int) high); + } else { + long top = high / EIGHT_DIGITS; + int middle = (int) (high - top * EIGHT_DIGITS); + pos = writePositiveInt(bytes, pos, (int) top); + pos = writePadded8Digits(bytes, pos, middle); + } + position = writePadded8Digits(bytes, pos, low); + } + + private static int writePositiveInt(byte[] bytes, int pos, int value) { if (value < 10000) { - position = writeIntUpTo4(bytes, pos, value); - return; + return writeIntUpTo4(bytes, pos, value); } int high = divide10000(value); int low = value - high * 10000; if (high < 10000) { if (high >= 1000) { - position = writePadded8(bytes, pos, high, low); - return; + return writePadded8(bytes, pos, high, low); } pos = writeIntUpTo4(bytes, pos, high); - position = writePadded4(bytes, pos, low); - return; + return writePadded4(bytes, pos, low); } int top = divide10000(high); int middle = high - top * 10000; pos = writeIntUpTo4(bytes, pos, top); - position = writePadded8(bytes, pos, middle, low); + return writePadded8(bytes, pos, middle, low); + } + + private static int writePadded8Digits(byte[] bytes, int pos, int value) { + int high = divide10000(value); + int low = value - high * 10000; + return writePadded8(bytes, pos, high, low); + } + + private static int writeHex(byte[] bytes, int pos, long value, int shift, int count) { + for (int i = 0; i < count; i++) { + bytes[pos++] = HEX_DIGITS[(int) ((value >>> shift) & 0xF)]; + shift -= 4; + } + return pos; + } + + private static int writeLocalDateBytes(byte[] bytes, int pos, int year, int month, int day) { + pos = writePadded4(bytes, pos, year); + bytes[pos++] = (byte) '-'; + pos = writeTwoDigits(bytes, pos, month); + bytes[pos++] = (byte) '-'; + return writeTwoDigits(bytes, pos, day); + } + + private static int writeTime(byte[] bytes, int pos, int hour, int minute, int second, int nano) { + pos = writeTwoDigits(bytes, pos, hour); + bytes[pos++] = (byte) ':'; + pos = writeTwoDigits(bytes, pos, minute); + if (second != 0 || nano != 0) { + bytes[pos++] = (byte) ':'; + pos = writeTwoDigits(bytes, pos, second); + if (nano != 0) { + bytes[pos++] = (byte) '.'; + pos = writeNano(bytes, pos, nano); + } + } + return pos; + } + + private static int writeNano(byte[] bytes, int pos, int nano) { + if (nano % 1_000_000 == 0) { + return writePadded3(bytes, pos, nano / 1_000_000); + } + if (nano % 1000 == 0) { + int micros = nano / 1000; + int high = micros / 1000; + int low = micros - high * 1000; + pos = writePadded3(bytes, pos, high); + return writePadded3(bytes, pos, low); + } + int first = nano / 100000000; + int rem = nano - first * 100000000; + int middle = rem / 10000; + int low = rem - middle * 10000; + bytes[pos++] = (byte) ('0' + first); + pos = writePadded4(bytes, pos, middle); + return writePadded4(bytes, pos, low); + } + + private static int writePadded3(byte[] bytes, int pos, int value) { + int high = value / 100; + int rem = value - high * 100; + int middle = rem / 10; + bytes[pos++] = (byte) ('0' + high); + bytes[pos++] = (byte) ('0' + middle); + bytes[pos++] = (byte) ('0' + (rem - middle * 10)); + return pos; + } + + private static int writeTwoDigits(byte[] bytes, int pos, int value) { + int high = value / 10; + bytes[pos++] = (byte) ('0' + high); + bytes[pos++] = (byte) ('0' + (value - high * 10)); + return pos; } private static int divide10000(int value) { diff --git a/java/fory-json/src/test/java/org/apache/fory/json/JsonContainerTest.java b/java/fory-json/src/test/java/org/apache/fory/json/JsonContainerTest.java index 43bec41c67..02c4a27551 100644 --- a/java/fory-json/src/test/java/org/apache/fory/json/JsonContainerTest.java +++ b/java/fory-json/src/test/java/org/apache/fory/json/JsonContainerTest.java @@ -31,6 +31,9 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.concurrent.atomic.AtomicIntegerArray; +import java.util.concurrent.atomic.AtomicLongArray; +import java.util.concurrent.atomic.AtomicReferenceArray; import org.apache.fory.json.data.FastContainers; import org.apache.fory.json.data.MapKeyFields; import org.apache.fory.json.data.Nested; @@ -119,6 +122,27 @@ public void parsedContainersStartSmall() { assertTrue(list instanceof ArrayList); assertEquals(arrayCapacity((ArrayList) list), 1); + List strings = + json.fromJson( + "[\"a\",\"b\",\"c\",\"d\",\"e\",\"f\",\"g\"]".getBytes(StandardCharsets.UTF_8), + new TypeRef>() {}); + assertTrue(strings instanceof ArrayList); + assertEquals(strings, Arrays.asList("a", "b", "c", "d", "e", "f", "g")); + assertEquals(arrayCapacity((ArrayList) strings), 7); + + List notes = + json.fromJson( + ("[{\"title\":\"a\"},{\"title\":\"b\"},null,{\"title\":\"d\"}," + + "{\"title\":\"e\"},{\"title\":\"f\"},{\"title\":\"g\"}]") + .getBytes(StandardCharsets.UTF_8), + new TypeRef>() {}); + assertTrue(notes instanceof ArrayList); + assertEquals(notes.size(), 7); + assertEquals(notes.get(0).title, "a"); + assertEquals(notes.get(2), null); + assertEquals(notes.get(6).title, "g"); + assertEquals(arrayCapacity((ArrayList) notes), 7); + JSONObject object = json.fromJson("{\"x\":1}", JSONObject.class); assertEquals(mapCapacity(object), 2); assertEquals(mapCapacity(json.fromJson("{}", JSONObject.class)), 0); @@ -261,6 +285,208 @@ public void readPrimitiveArrayRoots() { assertThrows(ForyJsonException.class, () -> json.fromJson("[1,null]", int[].class)); } + @Test + public void readUtf8StringArrays() { + ForyJson json = ForyJson.builder().build(); + assertEquals( + json.fromJson("[\"a\"]".getBytes(StandardCharsets.UTF_8), String[].class), + new String[] {"a"}); + assertEquals( + json.fromJson( + "[\"a\",null,\"b\",\"c\",\"d\",\"e\",\"f\",\"g\"]".getBytes(StandardCharsets.UTF_8), + String[].class), + new String[] {"a", null, "b", "c", "d", "e", "f", "g"}); + assertEquals( + json.fromJson( + "[\"a\",\"b\",\"c\",\"d\",\"e\",\"f\",\"g\",\"h\",\"i\"]" + .getBytes(StandardCharsets.UTF_8), + String[].class), + new String[] {"a", "b", "c", "d", "e", "f", "g", "h", "i"}); + } + + @Test + public void readStringInputStringArrays() { + ForyJson json = ForyJson.builder().build(); + assertEquals(json.fromJson("[\"a\"]", String[].class), new String[] {"a"}); + assertEquals( + json.fromJson("[\"a\",null,\"b\",\"c\",\"d\",\"e\",\"f\",\"g\"]", String[].class), + new String[] {"a", null, "b", "c", "d", "e", "f", "g"}); + assertEquals( + json.fromJson("[\"a\",\"b\",\"c\",\"d\",\"e\",\"f\",\"g\",\"h\",\"i\"]", String[].class), + new String[] {"a", "b", "c", "d", "e", "f", "g", "h", "i"}); + assertEquals( + json.fromJson( + "[\"a\",\"b\",\"c\",\"d\",\"e\",\"f\",\"g\",\"h\",\"i\",\"\u0100\"]", String[].class), + new String[] {"a", "b", "c", "d", "e", "f", "g", "h", "i", "\u0100"}); + } + + @Test + public void readUtf8LongArrays() { + ForyJson json = ForyJson.builder().build(); + assertEquals( + json.fromJson("[7]".getBytes(StandardCharsets.UTF_8), long[].class), new long[] {7L}); + assertEquals( + json.fromJson("[1,2,3,4,5,6,7,8]".getBytes(StandardCharsets.UTF_8), long[].class), + new long[] {1L, 2L, 3L, 4L, 5L, 6L, 7L, 8L}); + assertEquals( + json.fromJson("[1,2,3,4,5,6,7,8,9]".getBytes(StandardCharsets.UTF_8), long[].class), + new long[] {1L, 2L, 3L, 4L, 5L, 6L, 7L, 8L, 9L}); + assertThrows( + ForyJsonException.class, + () -> json.fromJson("[1,null]".getBytes(StandardCharsets.UTF_8), long[].class)); + } + + @Test + public void readStringInputLongArrays() { + ForyJson json = ForyJson.builder().build(); + assertEquals(json.fromJson("[7]", long[].class), new long[] {7L}); + assertEquals( + json.fromJson("[1,2,3,4,5,6,7,8]", long[].class), + new long[] {1L, 2L, 3L, 4L, 5L, 6L, 7L, 8L}); + assertEquals( + json.fromJson("[1,2,3,4,5,6,7,8,9]", long[].class), + new long[] {1L, 2L, 3L, 4L, 5L, 6L, 7L, 8L, 9L}); + LongArrayUtf16Root root = + json.fromJson( + "{\"values\":[1,2,3,4,5,6,7,8,9],\"text\":\"\u0100\"}", LongArrayUtf16Root.class); + assertEquals(root.values, new long[] {1L, 2L, 3L, 4L, 5L, 6L, 7L, 8L, 9L}); + assertEquals(root.text, "\u0100"); + assertThrows(ForyJsonException.class, () -> json.fromJson("[1,null]", long[].class)); + } + + @Test + public void writeReadBoxedPrimitiveArrays() { + ForyJson json = ForyJson.builder().build(); + assertEquals(json.toJson(new Integer[] {1, null, -2}), "[1,null,-2]"); + assertEquals( + json.fromJson("[1,null,-2]".getBytes(StandardCharsets.UTF_8), Integer[].class), + new Integer[] {1, null, -2}); + assertEquals( + json.fromJson("[9223372036854775807,null,-9]", Long[].class), + new Long[] {Long.MAX_VALUE, null, -9L}); + assertEquals( + json.fromJson("[true,null,false]".getBytes(StandardCharsets.UTF_8), Boolean[].class), + new Boolean[] {Boolean.TRUE, null, Boolean.FALSE}); + assertEquals( + json.fromJson("[32767,null,-32768]", Short[].class), + new Short[] {Short.MAX_VALUE, null, Short.MIN_VALUE}); + assertEquals( + json.fromJson("[127,null,-128]", Byte[].class), + new Byte[] {Byte.MAX_VALUE, null, Byte.MIN_VALUE}); + assertEquals( + json.fromJson("[\"a\",null,\"你\"]", Character[].class), new Character[] {'a', null, '你'}); + assertEquals( + json.fromJson("[1.5,null,-2.25]", Float[].class), new Float[] {1.5f, null, -2.25f}); + assertEquals( + json.fromJson("[1.5,null,-2.25]".getBytes(StandardCharsets.UTF_8), Double[].class), + new Double[] {1.5d, null, -2.25d}); + assertEquals( + new String(json.toJsonBytes(new Character[] {'x', null, '文'}), StandardCharsets.UTF_8), + "[\"x\",null,\"文\"]"); + + assertThrows(ForyJsonException.class, () -> json.fromJson("[128]", Byte[].class)); + assertThrows(ForyJsonException.class, () -> json.fromJson("[\"ab\"]", Character[].class)); + } + + @Test + public void readReferenceObjectArrays() { + ForyJson json = ForyJson.builder().build(); + Note[] notes = json.fromJson("[{\"title\":\"one\"},null,{\"title\":\"two\"}]", Note[].class); + assertEquals(notes.getClass(), Note[].class); + assertEquals(notes.length, 3); + assertEquals(notes[0].title, "one"); + assertEquals(notes[1], null); + assertEquals(notes[2].title, "two"); + + Note[] empty = json.fromJson("[]".getBytes(StandardCharsets.UTF_8), Note[].class); + assertEquals(empty.getClass(), Note[].class); + assertEquals(empty.length, 0); + + Note[][] grid = + json.fromJson( + "[[{\"title\":\"left\"}],null,[{\"title\":\"right\"},null]]" + .getBytes(StandardCharsets.UTF_8), + Note[][].class); + assertEquals(grid.getClass(), Note[][].class); + assertEquals(grid.length, 3); + assertEquals(grid[0].getClass(), Note[].class); + assertEquals(grid[0][0].title, "left"); + assertEquals(grid[1], null); + assertEquals(grid[2].getClass(), Note[].class); + assertEquals(grid[2][0].title, "right"); + assertEquals(grid[2][1], null); + } + + @Test + public void readReentrantObjectArray() { + ForyJson json = ForyJson.builder().build(); + ArrayNode[] roots = + json.fromJson( + "[{\"name\":\"root\",\"children\":[{\"name\":\"leaf\",\"children\":[]}]}]" + .getBytes(StandardCharsets.UTF_8), + ArrayNode[].class); + assertEquals(roots.getClass(), ArrayNode[].class); + assertEquals(roots.length, 1); + assertEquals(roots[0].name, "root"); + assertEquals(roots[0].children.getClass(), ArrayNode[].class); + assertEquals(roots[0].children.length, 1); + assertEquals(roots[0].children[0].name, "leaf"); + assertEquals(roots[0].children[0].children.getClass(), ArrayNode[].class); + assertEquals(roots[0].children[0].children.length, 0); + + ArrayNode[] deep = + json.fromJson(arrayNodeJson(9).getBytes(StandardCharsets.UTF_8), ArrayNode[].class); + assertEquals(deep.getClass(), ArrayNode[].class); + assertEquals(deep.length, 1); + ArrayNode node = deep[0]; + for (int i = 0; i < 9; i++) { + assertEquals(node.name, "node-" + i); + assertEquals(node.children.getClass(), ArrayNode[].class); + if (i == 8) { + assertEquals(node.children.length, 0); + } else { + assertEquals(node.children.length, 1); + node = node.children[0]; + } + } + } + + @Test + public void writeReadAtomicArrays() { + ForyJson json = ForyJson.builder().build(); + assertEquals(json.toJson(new AtomicIntegerArray(new int[] {1, -2, 3})), "[1,-2,3]"); + assertAtomicInts(json.fromJson("[1,-2,3]", AtomicIntegerArray.class), 1, -2, 3); + assertEquals( + new String( + json.toJsonBytes(new AtomicLongArray(new long[] {4L, -5L})), StandardCharsets.UTF_8), + "[4,-5]"); + assertAtomicLongs( + json.fromJson("[4,-5]".getBytes(StandardCharsets.UTF_8), AtomicLongArray.class), 4L, -5L); + + AtomicReferenceArray refs = + new AtomicReferenceArray<>(new String[] {"a", null, ZH_TEXT}); + assertEquals(json.toJson(refs), "[\"a\",null,\"你好，Fory\"]"); + assertAtomicRefs( + json.fromJson( + "[\"a\",null,\"你好，Fory\"]".getBytes(StandardCharsets.UTF_8), + new TypeRef>() {}), + "a", + null, + ZH_TEXT); + + String fieldsJson = "{\"ints\":[7,8],\"longs\":[9,-10],\"refs\":[\"b\",null]}"; + AtomicArrayFields fields = + json.fromJson(fieldsJson.getBytes(StandardCharsets.UTF_8), AtomicArrayFields.class); + assertAtomicInts(fields.ints, 7, 8); + assertAtomicLongs(fields.longs, 9L, -10L); + assertAtomicRefs(fields.refs, "b", null); + assertEquals(json.toJson(fields), fieldsJson); + + assertThrows( + ForyJsonException.class, () -> json.fromJson("[1,null]", AtomicIntegerArray.class)); + assertThrows(ForyJsonException.class, () -> json.fromJson("[1,null]", AtomicLongArray.class)); + } + public static final class Shelf { public NoteList notes; } @@ -271,7 +497,53 @@ public static final class Note { public String title; } + public static final class ArrayNode { + public String name; + public ArrayNode[] children; + } + public static final class PaletteGroups extends HashMap {} public static final class PaletteCodes extends HashMap {} + + public static final class AtomicArrayFields { + public AtomicIntegerArray ints = new AtomicIntegerArray(new int[] {7, 8}); + public AtomicLongArray longs = new AtomicLongArray(new long[] {9L, -10L}); + public AtomicReferenceArray refs = new AtomicReferenceArray<>(new String[] {"b", null}); + } + + public static final class LongArrayUtf16Root { + public long[] values; + public String text; + } + + private static String arrayNodeJson(int depth) { + return "[" + nodeJson(0, depth) + "]"; + } + + private static String nodeJson(int index, int depth) { + String children = index + 1 == depth ? "[]" : "[" + nodeJson(index + 1, depth) + "]"; + return "{\"name\":\"node-" + index + "\",\"children\":" + children + "}"; + } + + private static void assertAtomicInts(AtomicIntegerArray array, int... expected) { + assertEquals(array.length(), expected.length); + for (int i = 0; i < expected.length; i++) { + assertEquals(array.get(i), expected[i]); + } + } + + private static void assertAtomicLongs(AtomicLongArray array, long... expected) { + assertEquals(array.length(), expected.length); + for (int i = 0; i < expected.length; i++) { + assertEquals(array.get(i), expected[i]); + } + } + + private static void assertAtomicRefs(AtomicReferenceArray array, String... expected) { + assertEquals(array.length(), expected.length); + for (int i = 0; i < expected.length; i++) { + assertEquals(array.get(i), expected[i]); + } + } } diff --git a/java/fory-json/src/test/java/org/apache/fory/json/JsonGeneratedCodecTest.java b/java/fory-json/src/test/java/org/apache/fory/json/JsonGeneratedCodecTest.java index cf6a10d7d1..e693758f42 100644 --- a/java/fory-json/src/test/java/org/apache/fory/json/JsonGeneratedCodecTest.java +++ b/java/fory-json/src/test/java/org/apache/fory/json/JsonGeneratedCodecTest.java @@ -20,17 +20,33 @@ package org.apache.fory.json; import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertNotEquals; +import static org.testng.Assert.assertTrue; +import java.lang.reflect.Field; import java.nio.charset.StandardCharsets; +import java.util.ArrayList; import java.util.Arrays; +import java.util.concurrent.atomic.AtomicReference; +import org.apache.fory.json.codec.BaseObjectCodec; +import org.apache.fory.json.codec.GeneratedObjectCodec; import org.apache.fory.json.data.GeneratedCollectionFields; +import org.apache.fory.json.data.PublicFields; import org.apache.fory.json.data.RecursiveChild; import org.apache.fory.json.data.RecursiveParent; import org.apache.fory.json.data.TokenGroup; import org.apache.fory.json.data.TokenValues; +import org.apache.fory.json.meta.JsonAsciiToken; +import org.apache.fory.json.meta.JsonFieldNameHash; +import org.apache.fory.json.reader.Latin1JsonReader; +import org.apache.fory.json.reader.Utf8JsonReader; +import org.apache.fory.json.resolver.JsonTypeResolver; import org.testng.annotations.Test; public class JsonGeneratedCodecTest extends ForyJsonTestModels { + private static final String GENERATED_SUFFIX = "ForyJsonCodec"; + @Test public void writeRecursiveGeneratedTypes() { ForyJson json = ForyJson.builder().build(); @@ -54,7 +70,7 @@ public void writeGeneratedTokenChanges() { assertEquals(new String(json.toJsonBytes(value), StandardCharsets.UTF_8), first); value.count = 7; value.name = "beta"; - value.tags = Arrays.asList("z", "x"); + value.tags = new ArrayList<>(Arrays.asList("z", "x")); value.total = 9; String second = "{\"count\":7,\"name\":\"beta\",\"tags\":[\"z\",\"x\"],\"total\":9}"; assertEquals(json.toJson(value), second); @@ -119,4 +135,185 @@ public void readGeneratedCollectionFields() { json.fromJson(input.getBytes(StandardCharsets.UTF_8), GeneratedCollectionFields.class)); assertGeneratedWhenSupported(json, GeneratedCollectionFields.class); } + + @Test + public void sameConfigUsesSameId() throws Exception { + ForyJson first = ForyJson.builder().build(); + ForyJson second = ForyJson.builder().build(); + ForyJson writeNullFields = ForyJson.builder().writeNullFields(true).build(); + first.toJsonBytes(new PublicFields()); + second.toJsonBytes(new PublicFields()); + writeNullFields.toJsonBytes(new PublicFields()); + + Class firstWriterClass = generatedUtf8WriterClass(first, PublicFields.class); + Class secondWriterClass = generatedUtf8WriterClass(second, PublicFields.class); + Class writeNullWriterClass = generatedUtf8WriterClass(writeNullFields, PublicFields.class); + assertEquals( + firstWriterClass.getPackage().getName(), PublicFields.class.getPackage().getName()); + assertEquals( + secondWriterClass.getPackage().getName(), PublicFields.class.getPackage().getName()); + assertGeneratedName(firstWriterClass, PublicFields.class, "Utf8"); + assertGeneratedName(secondWriterClass, PublicFields.class, "Utf8"); + assertGeneratedName(writeNullWriterClass, PublicFields.class, "Utf8"); + assertEquals(generatedId(secondWriterClass), generatedId(firstWriterClass)); + assertNotEquals(generatedId(writeNullWriterClass), generatedId(firstWriterClass)); + } + + @Test + public void readLongAsciiFieldToken() { + String token = "\"favoriteFruit\":"; + long prefix = JsonAsciiToken.prefix(token); + long suffix = JsonAsciiToken.suffixLong(token); + long suffixMask = JsonAsciiToken.suffixMask(token.length()); + Utf8JsonReader utf8 = + new Utf8JsonReader((token + "\"apple\"").getBytes(StandardCharsets.UTF_8)); + assertTrue(utf8.tryReadNextFieldNameToken8(prefix, suffix, suffixMask, token.length())); + assertEquals(utf8.readNullableStringToken(), "apple"); + + Latin1JsonReader latin1 = new Latin1JsonReader(latin1Bytes(token + "\"pear\"")); + assertTrue(latin1.tryReadNextFieldNameToken8(prefix, suffix, suffixMask, token.length())); + assertEquals(latin1.readNullableStringToken(), "pear"); + + String tailToken = "\"registered\":"; + long tailPrefix = JsonAsciiToken.prefix(tailToken); + long tailSuffix = JsonAsciiToken.suffixLong(tailToken); + long tailSuffixMask = JsonAsciiToken.suffixMask(tailToken.length()); + Utf8JsonReader tailUtf8 = + new Utf8JsonReader((tailToken + "1").getBytes(StandardCharsets.UTF_8)); + assertTrue( + tailUtf8.tryReadNextFieldNameToken8( + tailPrefix, tailSuffix, tailSuffixMask, tailToken.length())); + assertEquals(tailUtf8.readIntTokenValue(), 1); + Latin1JsonReader tailLatin1 = new Latin1JsonReader(latin1Bytes(tailToken + "2")); + assertTrue( + tailLatin1.tryReadNextFieldNameToken8( + tailPrefix, tailSuffix, tailSuffixMask, tailToken.length())); + assertEquals(tailLatin1.readIntTokenValue(), 2); + + Utf8JsonReader mismatch = + new Utf8JsonReader("\"favoriteSeed\":\"pit\"".getBytes(StandardCharsets.UTF_8)); + assertFalse(mismatch.tryReadNextFieldNameToken8(prefix, suffix, suffixMask, token.length())); + assertEquals(mismatch.readFieldNameHash(), JsonFieldNameHash.hash("favoriteSeed")); + mismatch.expectNextToken(':'); + assertEquals(mismatch.readNextNullableString(), "pit"); + } + + @Test + public void readGeneratedLongAsciiFields() { + ForyJson json = ForyJson.builder().build(); + String input = + "{\"registered\":\"today\",\"longitude\":12.5,\"favoriteFruit\":\"apple\"," + + "\"shortName\":\"core\"}"; + assertLongAsciiFields(json.fromJson(input, LongAsciiFields.class)); + assertLongAsciiFields( + json.fromJson(input.getBytes(StandardCharsets.UTF_8), LongAsciiFields.class)); + assertGeneratedWhenSupported(json, LongAsciiFields.class); + } + + @Test + public void readSplitGeneratedFields() { + ForyJson json = ForyJson.builder().build(); + String ordered = + "{\"f0\":0,\"f1\":\"one\",\"f2\":2,\"f3\":\"three\",\"f4\":4,\"f5\":\"five\"," + + "\"f6\":6,\"f7\":\"seven\",\"f8\":8,\"f9\":\"nine\",\"f10\":10," + + "\"f11\":\"eleven\",\"f12\":12,\"f13\":\"thirteen\"}"; + assertWideFields(json.fromJson(ordered, WideFields.class)); + assertWideFields(json.fromJson(ordered.getBytes(StandardCharsets.UTF_8), WideFields.class)); + + String boundaryFallback = + "{\"f0\":0,\"f2\":2,\"f1\":\"one\",\"f3\":\"three\",\"f4\":4,\"f5\":\"five\"," + + "\"f6\":6,\"f7\":\"seven\",\"f8\":8,\"f9\":\"nine\",\"f10\":10," + + "\"f11\":\"eleven\",\"f12\":12,\"f13\":\"thirteen\"}"; + assertWideFields(json.fromJson(boundaryFallback, WideFields.class)); + assertWideFields( + json.fromJson(boundaryFallback.getBytes(StandardCharsets.UTF_8), WideFields.class)); + assertGeneratedWhenSupported(json, WideFields.class); + } + + private static void assertWideFields(WideFields value) { + assertEquals(value.f0, 0); + assertEquals(value.f1, "one"); + assertEquals(value.f2, 2); + assertEquals(value.f3, "three"); + assertEquals(value.f4, 4); + assertEquals(value.f5, "five"); + assertEquals(value.f6, 6); + assertEquals(value.f7, "seven"); + assertEquals(value.f8, 8); + assertEquals(value.f9, "nine"); + assertEquals(value.f10, 10); + assertEquals(value.f11, "eleven"); + assertEquals(value.f12, 12); + assertEquals(value.f13, "thirteen"); + } + + private static void assertLongAsciiFields(LongAsciiFields value) { + assertEquals(value.registered, "today"); + assertEquals(value.longitude, 12.5d); + assertEquals(value.favoriteFruit, "apple"); + assertEquals(value.shortName, "core"); + } + + private static byte[] latin1Bytes(String value) { + return value.getBytes(StandardCharsets.ISO_8859_1); + } + + public static class LongAsciiFields { + public String registered; + public double longitude; + public String favoriteFruit; + public String shortName; + } + + public static class WideFields { + public int f0; + public String f1; + public int f2; + public String f3; + public int f4; + public String f5; + public int f6; + public String f7; + public int f8; + public String f9; + public int f10; + public String f11; + public int f12; + public String f13; + } + + private static Class generatedUtf8WriterClass(ForyJson json, Class type) throws Exception { + Field primarySlotField = ForyJson.class.getDeclaredField("primarySlot"); + primarySlotField.setAccessible(true); + AtomicReference primarySlot = (AtomicReference) primarySlotField.get(json); + Object pooledState = primarySlot.get(); + Field stateField = pooledState.getClass().getDeclaredField("state"); + stateField.setAccessible(true); + Object state = stateField.get(pooledState); + Field typeResolverField = state.getClass().getDeclaredField("typeResolver"); + typeResolverField.setAccessible(true); + JsonTypeResolver typeResolver = (JsonTypeResolver) typeResolverField.get(state); + BaseObjectCodec codec = typeResolver.getObjectCodec(type); + assertTrue(codec instanceof GeneratedObjectCodec); + Field utf8WriterField = GeneratedObjectCodec.class.getDeclaredField("utf8Writer"); + utf8WriterField.setAccessible(true); + return utf8WriterField.get(codec).getClass(); + } + + private static void assertGeneratedName( + Class generatedClass, Class valueType, String role) { + String simpleName = generatedClass.getSimpleName(); + assertTrue(simpleName.startsWith(valueType.getSimpleName()), generatedClass.getName()); + assertTrue(simpleName.contains(role + GENERATED_SUFFIX), generatedClass.getName()); + assertFalse(simpleName.contains(GENERATED_SUFFIX + "_"), generatedClass.getName()); + assertTrue(generatedId(generatedClass) >= 0, generatedClass.getName()); + } + + private static int generatedId(Class generatedClass) { + String simpleName = generatedClass.getSimpleName(); + int suffixStart = simpleName.lastIndexOf(GENERATED_SUFFIX); + assertTrue(suffixStart >= 0, generatedClass.getName()); + String id = simpleName.substring(suffixStart + GENERATED_SUFFIX.length()); + return id.isEmpty() ? 0 : Integer.parseInt(id); + } } diff --git a/java/fory-json/src/test/java/org/apache/fory/json/JsonObjectTest.java b/java/fory-json/src/test/java/org/apache/fory/json/JsonObjectTest.java index 1cd303cb3d..3ed61ac7e4 100644 --- a/java/fory-json/src/test/java/org/apache/fory/json/JsonObjectTest.java +++ b/java/fory-json/src/test/java/org/apache/fory/json/JsonObjectTest.java @@ -20,7 +20,11 @@ package org.apache.fory.json; import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertThrows; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.OutputStream; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.List; @@ -48,6 +52,34 @@ public void writePublicFields() { "{\"active\":true,\"id\":7,\"name\":\"fory\"}"); } + @Test + public void writeJsonToOutputStream() { + ForyJson json = ForyJson.builder().build(); + ByteArrayOutputStream output = new ByteArrayOutputStream(); + json.writeJsonTo(new PublicFields(), output); + assertEquals( + new String(output.toByteArray(), StandardCharsets.UTF_8), + "{\"active\":true,\"id\":7,\"name\":\"fory\"}"); + + output.reset(); + json.writeJsonTo(null, output); + assertEquals(new String(output.toByteArray(), StandardCharsets.UTF_8), "null"); + + OutputStream failing = + new OutputStream() { + @Override + public void write(int b) throws IOException { + throw new IOException("closed"); + } + + @Override + public void write(byte[] b, int off, int len) throws IOException { + throw new IOException("closed"); + } + }; + assertThrows(ForyJsonException.class, () -> json.writeJsonTo(new PublicFields(), failing)); + } + @Test public void writeFirstIntGenerated() { ForyJson json = ForyJson.builder().build(); @@ -126,15 +158,15 @@ public void writeNullFields() { ForyJson json = ForyJson.builder().writeNullFields(true).build(); assertEquals( json.toJson(new PublicFields()), - "{\"active\":true,\"id\":7,\"missing\":null,\"name\":\"fory\"}"); + "{\"active\":true,\"id\":7,\"name\":\"fory\",\"missing\":null}"); } @Test - public void ignoreMethods() { - ForyJson json = ForyJson.builder().build(); + public void fieldOnlyModeIgnoresMethods() { + ForyJson json = ForyJson.builder().withFieldMode(true).build(); assertEquals( json.toJson(new MethodsIgnored()), - "{\"hidden\":\"hidden\",\"setterCalls\":0,\"value\":\"field\"}"); + "{\"setterCalls\":0,\"value\":\"field\",\"hidden\":\"hidden\"}"); MethodsIgnored value = json.fromJson("{\"hidden\":\"json\",\"value\":\"json\"}", MethodsIgnored.class); assertEquals(hiddenValue(value), "json"); diff --git a/java/fory-json/src/test/java/org/apache/fory/json/JsonPropertyTest.java b/java/fory-json/src/test/java/org/apache/fory/json/JsonPropertyTest.java new file mode 100644 index 0000000000..ca3991dcfe --- /dev/null +++ b/java/fory-json/src/test/java/org/apache/fory/json/JsonPropertyTest.java @@ -0,0 +1,131 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.json; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertThrows; + +import org.apache.fory.json.data.BeanProperties.BooleanBean; +import org.apache.fory.json.data.BeanProperties.ConflictingTypesBean; +import org.apache.fory.json.data.BeanProperties.DuplicateGetterBean; +import org.apache.fory.json.data.BeanProperties.FinalFieldBean; +import org.apache.fory.json.data.BeanProperties.GetterBean; +import org.apache.fory.json.data.BeanProperties.GetterOnlyBean; +import org.apache.fory.json.data.BeanProperties.InheritedChild; +import org.apache.fory.json.data.BeanProperties.InheritedParent; +import org.apache.fory.json.data.BeanProperties.InvalidAccessorBean; +import org.apache.fory.json.data.BeanProperties.MixedBean; +import org.apache.fory.json.data.BeanProperties.OverloadedSetterBean; +import org.apache.fory.json.data.BeanProperties.SetterBean; +import org.apache.fory.json.data.BeanProperties.SetterOnlyBean; +import org.testng.annotations.Test; + +public class JsonPropertyTest extends ForyJsonTestModels { + @Test + public void writePrivateGetters() { + ForyJson json = ForyJson.builder().build(); + assertEquals(json.toJson(new GetterBean()), "{\"id\":17,\"name\":\"getter-field\"}"); + assertGeneratedWhenSupported(json, GetterBean.class); + } + + @Test + public void readPrivateSetters() { + ForyJson json = ForyJson.builder().build(); + SetterBean value = json.fromJson("{\"id\":4,\"name\":\"alpha\"}", SetterBean.class); + assertEquals(SetterBean.id(value), 5); + assertEquals(SetterBean.name(value), "set-alpha"); + assertEquals(SetterBean.setterCalls(value), 2); + assertGeneratedWhenSupported(json, SetterBean.class); + } + + @Test + public void roundTripMixedProperties() { + ForyJson json = ForyJson.builder().build(); + String text = "{\"count\":3,\"name\":\"mixed\",\"score\":5}"; + assertEquals(json.toJson(new MixedBean()), text); + assertEquals(json.toJson(json.fromJson(text, MixedBean.class)), text); + assertGeneratedWhenSupported(json, MixedBean.class); + } + + @Test + public void writeBooleanIsGetter() { + ForyJson json = ForyJson.builder().build(); + assertEquals(json.toJson(new BooleanBean()), "{\"ready\":true}"); + } + + @Test + public void getterOnlyWrites() { + ForyJson json = ForyJson.builder().build(); + assertEquals(json.toJson(new GetterOnlyBean()), "{\"computed\":6}"); + GetterOnlyBean value = json.fromJson("{\"computed\":99}", GetterOnlyBean.class); + assertEquals(json.toJson(value), "{\"computed\":6}"); + } + + @Test + public void setterOnlyReads() { + ForyJson json = ForyJson.builder().build(); + assertEquals(json.toJson(new SetterOnlyBean()), "{}"); + SetterOnlyBean value = json.fromJson("{\"secret\":\"alpha\"}", SetterOnlyBean.class); + assertEquals(SetterOnlyBean.received(value), "set-alpha"); + } + + @Test + public void fieldOnlyMode() { + ForyJson json = ForyJson.builder().withFieldMode(true).build(); + assertEquals(json.toJson(new GetterBean()), "{\"id\":7,\"name\":\"field\"}"); + assertEquals(json.toJson(new GetterOnlyBean()), "{}"); + SetterOnlyBean value = json.fromJson("{\"secret\":\"alpha\"}", SetterOnlyBean.class); + assertEquals(SetterOnlyBean.received(value), null); + } + + @Test + public void rejectPropertyConflicts() { + ForyJson json = ForyJson.builder().build(); + assertThrows(ForyJsonException.class, () -> json.toJson(new DuplicateGetterBean())); + assertThrows( + ForyJsonException.class, + () -> json.fromJson("{\"value\":\"alpha\"}", OverloadedSetterBean.class)); + assertThrows(ForyJsonException.class, () -> json.toJson(new ConflictingTypesBean())); + } + + @Test + public void inheritedProperties() { + ForyJson json = ForyJson.builder().build(); + assertEquals(json.toJson(new InheritedChild()), "{\"id\":4,\"name\":\"child\"}"); + InheritedChild value = json.fromJson("{\"id\":7,\"name\":\"json\"}", InheritedChild.class); + assertEquals(InheritedParent.id(value), 8); + assertEquals(value.name, "json"); + } + + @Test + public void ignoreInvalidAccessors() { + ForyJson json = ForyJson.builder().build(); + assertEquals(json.toJson(new InvalidAccessorBean()), "{\"value\":\"field\"}"); + } + + @Test + public void finalFieldsStayReadOnly() { + ForyJson json = ForyJson.builder().build(); + assertEquals(json.toJson(new FinalFieldBean()), "{\"id\":1,\"name\":\"field\"}"); + FinalFieldBean value = json.fromJson("{\"id\":9,\"name\":\"json\"}", FinalFieldBean.class); + assertEquals(value.id, 1); + assertEquals(value.name, "json"); + } +} diff --git a/java/fory-json/src/test/java/org/apache/fory/json/JsonRecordTest.java b/java/fory-json/src/test/java/org/apache/fory/json/JsonRecordTest.java index b65328b5bc..d2b23f264f 100644 --- a/java/fory-json/src/test/java/org/apache/fory/json/JsonRecordTest.java +++ b/java/fory-json/src/test/java/org/apache/fory/json/JsonRecordTest.java @@ -50,7 +50,7 @@ public void writeReadRecordClass() throws Exception { .newInstance(7, ZH_TEXT, Arrays.asList("a", "b"), child); ForyJson json = ForyJson.builder().build(); String expected = - "{\"child\":{\"label\":\"kid\"},\"id\":7,\"name\":\"你好，Fory\"," + "\"tags\":[\"a\",\"b\"]}"; + "{\"id\":7,\"name\":\"你好，Fory\",\"tags\":[\"a\",\"b\"]," + "\"child\":{\"label\":\"kid\"}}"; assertEquals(json.toJson(value), expected); assertEquals(new String(json.toJsonBytes(value), StandardCharsets.UTF_8), expected); assertGeneratedWhenSupported(json, type); diff --git a/java/fory-json/src/test/java/org/apache/fory/json/JsonScalarTest.java b/java/fory-json/src/test/java/org/apache/fory/json/JsonScalarTest.java index 84c1f8afbf..339bd7d879 100644 --- a/java/fory-json/src/test/java/org/apache/fory/json/JsonScalarTest.java +++ b/java/fory-json/src/test/java/org/apache/fory/json/JsonScalarTest.java @@ -23,19 +23,39 @@ import static org.testng.Assert.assertThrows; import java.io.File; +import java.math.BigDecimal; import java.math.BigInteger; import java.nio.charset.StandardCharsets; import java.nio.file.Path; import java.nio.file.Paths; import java.time.LocalDate; import java.time.LocalDateTime; +import java.time.OffsetDateTime; +import java.time.ZoneOffset; import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; +import org.apache.fory.json.codec.JsonCodec; import org.apache.fory.json.data.BoxedScalars; import org.apache.fory.json.data.CoreScalarFields; import org.apache.fory.json.data.NaturalObjectValue; import org.apache.fory.json.data.NaturalValues; import org.apache.fory.json.data.NumericBoundaries; import org.apache.fory.json.data.PublicFields; +import org.apache.fory.json.reader.JsonReader; +import org.apache.fory.json.reader.Latin1JsonReader; +import org.apache.fory.json.reader.Utf16JsonReader; +import org.apache.fory.json.reader.Utf8JsonReader; +import org.apache.fory.json.resolver.JsonTypeInfo; +import org.apache.fory.json.resolver.JsonTypeResolver; +import org.apache.fory.json.writer.JsonWriter; +import org.apache.fory.json.writer.StringJsonWriter; +import org.apache.fory.json.writer.Utf8JsonWriter; +import org.apache.fory.reflect.TypeRef; +import org.apache.fory.serializer.StringSerializer; import org.testng.annotations.Test; public class JsonScalarTest extends ForyJsonTestModels { @@ -50,6 +70,32 @@ public void writeBoxedScalars() { new String(json.toJsonBytes(new BoxedScalars()), StandardCharsets.UTF_8), expected); } + @Test + public void writeUtf8Doubles() { + double[] values = { + 0.0d, + -0.0d, + 2.5d, + -3.75d, + Math.nextUp(1.0d), + 32.389082173209815d, + 69.85922221416756d, + 1.0e-4d, + 1.0e-3d, + 1.0e7d, + Double.MIN_VALUE, + Double.MAX_VALUE + }; + for (double value : values) { + Utf8JsonWriter writer = new Utf8JsonWriter(false, new byte[4]); + writer.writeDouble(value); + assertEquals( + new String(writer.toJsonBytes(), StandardCharsets.UTF_8), Double.toString(value)); + } + Utf8JsonWriter writer = new Utf8JsonWriter(false, new byte[4]); + assertThrows(ForyJsonException.class, () -> writer.writeDouble(Double.NaN)); + } + @Test public void writeNaturalObjectValues() { ForyJson json = ForyJson.builder().build(); @@ -115,6 +161,144 @@ public void readNumericBoundaries() { "{\"intMax\":2147483648,\"text\":\"" + ZH_TEXT + "\"}", NumericBoundaries.class)); } + @Test + public void readUtf8DoubleTokens() { + assertEquals( + new Utf8JsonReader("12.375".getBytes(StandardCharsets.UTF_8)).readDoubleTokenValue(), + 12.375d); + assertEquals( + Double.doubleToRawLongBits( + new Utf8JsonReader("-0.0".getBytes(StandardCharsets.UTF_8)).readDoubleTokenValue()), + Double.doubleToRawLongBits(-0.0d)); + assertEquals( + new Utf8JsonReader("1.25e2".getBytes(StandardCharsets.UTF_8)).readDoubleTokenValue(), + 125.0d); + assertThrows( + ForyJsonException.class, + () -> new Utf8JsonReader("01.5".getBytes(StandardCharsets.UTF_8)).readDoubleTokenValue()); + } + + @Test + public void readLatin1DoubleTokens() { + assertEquals(new Latin1JsonReader(latin1Bytes("12.375")).readDouble(), 12.375d); + assertEquals( + Double.doubleToRawLongBits(new Latin1JsonReader(latin1Bytes("-0.0")).readDouble()), + Double.doubleToRawLongBits(-0.0d)); + assertEquals(new Latin1JsonReader(latin1Bytes("1.25e2")).readDouble(), 125.0d); + assertThrows( + ForyJsonException.class, () -> new Latin1JsonReader(latin1Bytes("01.5")).readDouble()); + } + + @Test + public void readUtf8LongBlocks() { + assertEquals( + new Utf8JsonReader("123456789012345678".getBytes(StandardCharsets.UTF_8)) + .readLongTokenValue(), + 123456789012345678L); + assertEquals( + new Utf8JsonReader("-123456789012345678".getBytes(StandardCharsets.UTF_8)) + .readLongTokenValue(), + -123456789012345678L); + assertEquals( + new Utf8JsonReader("9223372036854775807".getBytes(StandardCharsets.UTF_8)) + .readLongTokenValue(), + Long.MAX_VALUE); + assertEquals( + new Utf8JsonReader("-9223372036854775808".getBytes(StandardCharsets.UTF_8)) + .readLongTokenValue(), + Long.MIN_VALUE); + assertThrows( + ForyJsonException.class, + () -> + new Utf8JsonReader("9223372036854775808".getBytes(StandardCharsets.UTF_8)) + .readLongTokenValue()); + } + + @Test + public void readLatin1LongBlocks() { + assertEquals( + new Latin1JsonReader(latin1Bytes("123456789012345678")).readLongTokenValue(), + 123456789012345678L); + assertEquals( + new Latin1JsonReader(latin1Bytes("-123456789012345678")).readLongTokenValue(), + -123456789012345678L); + assertEquals( + new Latin1JsonReader(latin1Bytes("9223372036854775807")).readLongTokenValue(), + Long.MAX_VALUE); + assertEquals( + new Latin1JsonReader(latin1Bytes("-9223372036854775808")).readLongTokenValue(), + Long.MIN_VALUE); + assertThrows( + ForyJsonException.class, + () -> new Latin1JsonReader(latin1Bytes("9223372036854775808")).readLongTokenValue()); + assertThrows( + ForyJsonException.class, + () -> new Latin1JsonReader(latin1Bytes("-9223372036854775809")).readLongTokenValue()); + } + + @Test + public void writeNumericBoundaries() { + ForyJson json = ForyJson.builder().build(); + NumericBoundaries value = new NumericBoundaries(); + value.intMax = Integer.MAX_VALUE; + value.intMin = Integer.MIN_VALUE; + value.longMax = Long.MAX_VALUE; + value.longMin = Long.MIN_VALUE; + value.small = -7; + value.text = "ok"; + String expected = + "{\"intMax\":2147483647,\"intMin\":-2147483648," + + "\"longMax\":9223372036854775807,\"longMin\":-9223372036854775808," + + "\"small\":-7,\"text\":\"ok\"}"; + assertEquals(json.toJson(value), expected); + assertEquals(new String(json.toJsonBytes(value), StandardCharsets.UTF_8), expected); + } + + @Test + public void writeUtf16Numbers() { + StringJsonWriter writer = new StringJsonWriter(false); + writer.writeArrayStart(); + writer.writeString(ZH_TEXT); + writer.writeComma(1); + writer.writeLong(Long.MAX_VALUE); + writer.writeComma(2); + writer.writeLong(2_147_483_648L); + writer.writeComma(3); + writer.writeInt(Integer.MIN_VALUE); + writer.writeArrayEnd(); + assertEquals( + writer.toJson(), "[\"" + ZH_TEXT + "\",9223372036854775807,2147483648,-2147483648]"); + + ForyJson json = ForyJson.builder().build(); + Utf16NumericFields value = new Utf16NumericFields(); + value.prefix = ZH_TEXT; + value.zero = 0; + value.one = 7; + value.twoDigits = 42; + value.threeDigits = 321; + value.fourDigits = 9999; + value.fiveDigits = 10000; + value.eightDigits = 99999999; + value.nineDigits = 100000000; + value.intMax = Integer.MAX_VALUE; + value.intMin = Integer.MIN_VALUE; + value.aroundIntMax = 2_147_483_648L; + value.longMax = Long.MAX_VALUE; + value.longMin = Long.MIN_VALUE; + value.negative = -12345; + String expected = + "{\"prefix\":\"" + + ZH_TEXT + + "\",\"zero\":0,\"one\":7,\"twoDigits\":42,\"threeDigits\":321," + + "\"fourDigits\":9999,\"fiveDigits\":10000,\"eightDigits\":99999999," + + "\"nineDigits\":100000000,\"intMax\":2147483647," + + "\"intMin\":-2147483648,\"aroundIntMax\":2147483648," + + "\"longMax\":9223372036854775807,\"longMin\":-9223372036854775808," + + "\"negative\":-12345}"; + assertEquals(json.toJson(value), expected); + assertEquals(new String(json.toJsonBytes(value), StandardCharsets.UTF_8), expected); + } + @Test public void writeReadCoreScalarFields() { ForyJson json = ForyJson.builder().build(); @@ -150,15 +334,143 @@ public void writeReadCoreScalarFields() { assertEquals(read.uuid, value.uuid); } + @Test + public void writeReadAtomicScalars() { + ForyJson json = ForyJson.builder().build(); + assertEquals(json.toJson(new AtomicBoolean(true)), "true"); + assertEquals(json.fromJson("false", AtomicBoolean.class).get(), false); + assertEquals(json.toJson(new AtomicInteger(12)), "12"); + assertEquals(json.fromJson("13", AtomicInteger.class).get(), 13); + assertEquals(json.toJson(new AtomicLong(14L)), "14"); + assertEquals(json.fromJson("15", AtomicLong.class).get(), 15L); + assertEquals(json.toJson(new AtomicReference<>("value")), "\"value\""); + + AtomicReference value = + json.fromJson("\"typed\"", new TypeRef>() {}); + assertEquals(value.get(), "typed"); + AtomicReference nullValue = + json.fromJson("null", new TypeRef>() {}); + assertEquals(nullValue.get(), null); + } + + @Test + public void writeUtf8ScalarFormats() { + ForyJson json = ForyJson.builder().build(); + UUID uuid = UUID.fromString("123e4567-e89b-12d3-a456-426614174000"); + assertEquals( + new String(json.toJsonBytes(uuid), StandardCharsets.UTF_8), + "\"123e4567-e89b-12d3-a456-426614174000\""); + assertEquals( + new String(json.toJsonBytes(LocalDate.of(2024, 2, 3)), StandardCharsets.UTF_8), + "\"2024-02-03\""); + + OffsetDateTimeFields fields = new OffsetDateTimeFields(); + fields.value = OffsetDateTime.of(2024, 2, 3, 4, 5, 0, 0, ZoneOffset.UTC); + assertEquals( + new String(json.toJsonBytes(fields), StandardCharsets.UTF_8), + "{\"value\":\"2024-02-03T04:05Z\"}"); + fields.value = OffsetDateTime.of(2024, 2, 3, 4, 5, 0, 1_000_000, ZoneOffset.UTC); + assertEquals( + new String(json.toJsonBytes(fields), StandardCharsets.UTF_8), + "{\"value\":\"2024-02-03T04:05:00.001Z\"}"); + fields.value = OffsetDateTime.of(2024, 2, 3, 4, 5, 6, 120_000_000, ZoneOffset.UTC); + assertEquals( + new String(json.toJsonBytes(fields), StandardCharsets.UTF_8), + "{\"value\":\"2024-02-03T04:05:06.120Z\"}"); + fields.value = OffsetDateTime.of(2024, 2, 3, 4, 5, 6, 123_400_000, ZoneOffset.UTC); + assertEquals( + new String(json.toJsonBytes(fields), StandardCharsets.UTF_8), + "{\"value\":\"2024-02-03T04:05:06.123400Z\"}"); + fields.value = OffsetDateTime.of(2024, 2, 3, 4, 5, 6, 1_000, ZoneOffset.UTC); + assertEquals( + new String(json.toJsonBytes(fields), StandardCharsets.UTF_8), + "{\"value\":\"2024-02-03T04:05:06.000001Z\"}"); + fields.value = OffsetDateTime.of(2024, 2, 3, 4, 5, 6, 123456789, ZoneOffset.UTC); + assertEquals( + new String(json.toJsonBytes(fields), StandardCharsets.UTF_8), + "{\"value\":\"2024-02-03T04:05:06.123456789Z\"}"); + + OffsetDateTime offset = + OffsetDateTime.of(2024, 2, 3, 4, 5, 6, 123456789, ZoneOffset.ofHoursMinutes(8, 30)); + fields.value = offset; + assertEquals( + new String(json.toJsonBytes(fields), StandardCharsets.UTF_8), + "{\"value\":\"" + offset + "\"}"); + } + + @Test + public void writeGeneratedUtf8Scalars() { + ForyJson json = ForyJson.builder().build(); + Utf8ScalarFields fields = new Utf8ScalarFields(); + fields.uuid = UUID.fromString("123e4567-e89b-12d3-a456-426614174000"); + fields.decimal = new BigDecimal("12345.6789"); + fields.date = LocalDate.of(2024, 2, 3); + fields.timestamp = OffsetDateTime.of(2024, 2, 3, 4, 5, 6, 123456789, ZoneOffset.UTC); + String expected = + "{\"uuid\":\"123e4567-e89b-12d3-a456-426614174000\"," + + "\"decimal\":12345.6789," + + "\"date\":\"2024-02-03\"," + + "\"timestamp\":\"2024-02-03T04:05:06.123456789Z\"}"; + assertEquals(new String(json.toJsonBytes(fields), StandardCharsets.UTF_8), expected); + assertEquals(json.toJson(fields), expected); + assertGeneratedWhenSupported(json, Utf8ScalarFields.class); + } + @Test public void readScalarRoots() { ForyJson json = ForyJson.builder().build(); assertEquals(json.fromJson("7", int.class), Integer.valueOf(7)); assertEquals(json.fromJson("true", boolean.class), Boolean.TRUE); + assertEquals(json.fromJson("0.100", BigDecimal.class), new BigDecimal("0.100")); assertEquals(json.fromJson("\"fory\"".getBytes(StandardCharsets.UTF_8), String.class), "fory"); assertEquals( json.fromJson("\"\uD83D\uDE00\u1234\"".getBytes(StandardCharsets.UTF_8), String.class), "\uD83D\uDE00\u1234"); + assertEquals( + json.fromJson("0.100".getBytes(StandardCharsets.UTF_8), BigDecimal.class), + new BigDecimal("0.100")); + assertEquals( + json.fromJson( + "12345678901234567890.123".getBytes(StandardCharsets.UTF_8), BigDecimal.class), + new BigDecimal("12345678901234567890.123")); + assertEquals( + json.fromJson( + "\"123e4567-e89b-12d3-a456-426614174000\"".getBytes(StandardCharsets.UTF_8), + UUID.class), + UUID.fromString("123e4567-e89b-12d3-a456-426614174000")); + assertEquals( + json.fromJson("\"123e4567-e89b-12d3-a456-426614174000\"", UUID.class), + UUID.fromString("123e4567-e89b-12d3-a456-426614174000")); + } + + @Test + public void readGeneratedUtf8BigDecimal() { + ForyJson json = ForyJson.builder().build(); + byte[] input = + ("{\"uuid\":\"123e4567-e89b-12d3-a456-426614174000\"," + + "\"decimal\":0.12345678901234567," + + "\"date\":\"2024-02-03\"," + + "\"timestamp\":\"2024-02-03T04:05:06Z\"}") + .getBytes(StandardCharsets.UTF_8); + Utf8ScalarFields fields = json.fromJson(input, Utf8ScalarFields.class); + assertEquals(fields.uuid, UUID.fromString("123e4567-e89b-12d3-a456-426614174000")); + assertEquals(fields.decimal, new BigDecimal("0.12345678901234567")); + } + + @Test + public void readGeneratedLatin1Scalars() { + ForyJson json = ForyJson.builder().build(); + String input = + "{\"uuid\":\"123e4567-e89b-12d3-a456-426614174000\"," + + "\"decimal\":0.12345678901234567," + + "\"date\":\"2024-02-03\"," + + "\"timestamp\":\"2024-02-03T04:05:06.123456789Z\"}"; + Utf8ScalarFields fields = json.fromJson(input, Utf8ScalarFields.class); + assertEquals(fields.uuid, UUID.fromString("123e4567-e89b-12d3-a456-426614174000")); + assertEquals(fields.decimal, new BigDecimal("0.12345678901234567")); + assertEquals(fields.date, LocalDate.of(2024, 2, 3)); + assertEquals( + fields.timestamp, OffsetDateTime.of(2024, 2, 3, 4, 5, 6, 123456789, ZoneOffset.UTC)); } @Test @@ -225,11 +537,100 @@ public void readLocalDateFromDateTime() { ForyJson json = ForyJson.builder().build(); LocalDate expected = LocalDate.of(2023, 7, 2); assertEquals(json.fromJson("\"2023-07-02T16:00:00.000Z\"", LocalDate.class), expected); + assertEquals( + json.fromJson( + "\"2023-07-02T16:00:00.000Z\"".getBytes(StandardCharsets.UTF_8), LocalDate.class), + expected); LocalDateFields fields = json.fromJson("{\"value\":\"2023-07-02T16:00:00.000Z\"}", LocalDateFields.class); assertEquals(fields.value, expected); } + @Test + public void readOffsetDateTime() { + ForyJson json = ForyJson.builder().build(); + OffsetDateTime utc = OffsetDateTime.of(2024, 2, 3, 4, 5, 6, 0, ZoneOffset.UTC); + assertEquals(json.fromJson("\"2024-02-03T04:05:06Z\"", OffsetDateTime.class), utc); + assertEquals( + json.fromJson( + "\"2024-02-03T04:05:06\\u005A\"".getBytes(StandardCharsets.UTF_8), + OffsetDateTime.class), + utc); + + OffsetDateTime nanos = + OffsetDateTime.of(2024, 2, 3, 4, 5, 6, 123456789, ZoneOffset.ofHoursMinutes(8, 30)); + assertEquals( + json.fromJson( + "\"2024-02-03T04:05:06.123456789+08:30\"".getBytes(StandardCharsets.UTF_8), + OffsetDateTime.class), + nanos); + + OffsetDateTime minutePrecision = + OffsetDateTime.of(2024, 2, 3, 4, 5, 0, 0, ZoneOffset.ofHoursMinutes(-5, -30)); + OffsetDateTimeFields fields = + json.fromJson("{\"value\":\"2024-02-03T04:05-05:30\"}", OffsetDateTimeFields.class); + assertEquals(fields.value, minutePrecision); + } + + @Test + public void readUtf16TemporalScalars() { + LocalDate date = LocalDate.of(2023, 7, 2); + Utf16JsonReader dateReader = utf16Reader("\"2023-07-02T16:00:00.000Z\""); + assertEquals(dateReader.readIsoLocalDate(), date); + dateReader.finish(); + + OffsetDateTime timestamp = + OffsetDateTime.of(2024, 2, 3, 4, 5, 6, 123456789, ZoneOffset.ofHoursMinutes(8, 30)); + Utf16JsonReader timestampReader = utf16Reader("\"2024-02-03T04:05:06.123456789+08:30\""); + assertEquals(timestampReader.readIsoOffsetDateTime(), timestamp); + timestampReader.finish(); + + ForyJson json = ForyJson.builder().build(); + Utf16TemporalFields fields = + json.fromJson( + "{\"text\":\"中文\",\"date\":\"2023-07-02\"," + + "\"timestamp\":\"2024-02-03T04:05:06.123456789+08:30\"}", + Utf16TemporalFields.class); + assertEquals(fields.text, "中文"); + assertEquals(fields.date, date); + assertEquals(fields.timestamp, timestamp); + } + + @Test + public void objectFieldUsesUtf8Codec() { + ForyJson json = + ForyJson.builder().registerCodec(ModeAwareValue.class, new ModeAwareCodec()).build(); + ModeAwareHolder holder = + json.fromJson("{\"value\":{}}".getBytes(StandardCharsets.UTF_8), ModeAwareHolder.class); + assertEquals(holder.value.mode, "utf8"); + } + + @Test + public void byteInputUsesUtf8Codec() { + ForyJson json = + ForyJson.builder().registerCodec(ModeAwareValue.class, new ModeAwareCodec()).build(); + ModeAwareValue value = + json.fromJson("{}".getBytes(StandardCharsets.UTF_8), ModeAwareValue.class); + assertEquals(value.mode, "utf8"); + } + + @Test + public void stringInputUsesLatin1Codec() { + ForyJson json = + ForyJson.builder().registerCodec(ModeAwareValue.class, new ModeAwareCodec()).build(); + ModeAwareValue value = json.fromJson("{}", ModeAwareValue.class); + String expected = StringSerializer.isBytesBackedString() ? "latin1" : "utf16"; + assertEquals(value.mode, expected); + } + + @Test + public void stringInputUsesUtf16Codec() { + ForyJson json = + ForyJson.builder().registerCodec(ModeAwareValue.class, new ModeAwareCodec()).build(); + ModeAwareValue value = json.fromJson("{\"ignored\":\"\u0100\"}", ModeAwareValue.class); + assertEquals(value.mode, "utf16"); + } + @Test public void wrapStringScalarParseErrors() { ForyJson json = ForyJson.builder().build(); @@ -251,6 +652,10 @@ public static final class ClassFieldHolder { public Class type = String.class; } + private static byte[] latin1Bytes(String value) { + return value.getBytes(StandardCharsets.ISO_8859_1); + } + public static final class ClassArrayFields { public Class[] types = new Class[] {String.class}; } @@ -263,4 +668,101 @@ public static final class FilePathFields { public static final class LocalDateFields { public LocalDate value; } + + public static final class OffsetDateTimeFields { + public OffsetDateTime value; + } + + public static final class Utf8ScalarFields { + public UUID uuid; + public BigDecimal decimal; + public LocalDate date; + public OffsetDateTime timestamp; + } + + public static final class Utf16TemporalFields { + public String text; + public LocalDate date; + public OffsetDateTime timestamp; + } + + public static final class Utf16NumericFields { + public String prefix; + public int zero; + public int one; + public int twoDigits; + public int threeDigits; + public int fourDigits; + public int fiveDigits; + public int eightDigits; + public int nineDigits; + public int intMax; + public int intMin; + public long aroundIntMax; + public long longMax; + public long longMin; + public int negative; + } + + public static final class ModeAwareHolder { + public ModeAwareValue value; + } + + public static final class ModeAwareValue { + public final String mode; + + ModeAwareValue(String mode) { + this.mode = mode; + } + } + + private static final class ModeAwareCodec implements JsonCodec { + @Override + public void write(JsonWriter writer, Object value, JsonTypeResolver resolver) { + writer.writeNull(); + } + + @Override + public void writeString(StringJsonWriter writer, Object value, JsonTypeResolver resolver) { + writer.writeNull(); + } + + @Override + public void writeUtf8(Utf8JsonWriter writer, Object value, JsonTypeResolver resolver) { + writer.writeNull(); + } + + @Override + public Object read(JsonReader reader, JsonTypeInfo typeInfo, JsonTypeResolver resolver) { + reader.skipValue(); + return new ModeAwareValue("generic"); + } + + @Override + public Object readLatin1( + Latin1JsonReader reader, JsonTypeInfo typeInfo, JsonTypeResolver resolver) { + reader.skipValue(); + return new ModeAwareValue("latin1"); + } + + @Override + public Object readUtf16( + Utf16JsonReader reader, JsonTypeInfo typeInfo, JsonTypeResolver resolver) { + reader.skipValue(); + return new ModeAwareValue("utf16"); + } + + @Override + public Object readUtf8( + Utf8JsonReader reader, JsonTypeInfo typeInfo, JsonTypeResolver resolver) { + reader.skipValue(); + return new ModeAwareValue("utf8"); + } + } + + private static Utf16JsonReader utf16Reader(String input) { + byte[] bytes = new byte[input.length() << 1]; + StringSerializer.copyStringCharsToBytes(input, bytes); + return new Utf16JsonReader().resetUtf16Bytes(input, bytes); + } } diff --git a/java/fory-json/src/test/java/org/apache/fory/json/JsonStringTest.java b/java/fory-json/src/test/java/org/apache/fory/json/JsonStringTest.java index 823873b160..3caf82909b 100644 --- a/java/fory-json/src/test/java/org/apache/fory/json/JsonStringTest.java +++ b/java/fory-json/src/test/java/org/apache/fory/json/JsonStringTest.java @@ -20,6 +20,7 @@ package org.apache.fory.json; import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertThrows; import static org.testng.Assert.assertTrue; @@ -34,7 +35,11 @@ import org.apache.fory.json.data.UnicodeKind; import org.apache.fory.json.data.UnicodeMatrix; import org.apache.fory.json.data.UnicodeValues; +import org.apache.fory.json.meta.JsonFieldNameHash; +import org.apache.fory.json.reader.Latin1JsonReader; +import org.apache.fory.json.reader.Utf16JsonReader; import org.apache.fory.json.writer.StringJsonWriter; +import org.apache.fory.memory.NativeByteOrder; import org.apache.fory.serializer.StringSerializer; import org.testng.annotations.Test; @@ -63,6 +68,91 @@ public void writeUtf16StringText() { assertEquals(json.fromJson(json.toJsonBytes(values), UnicodeValues.class).tags, values.tags); } + @Test + public void readCachedUtf16Bytes() { + String input = "\"music \uD834\uDD1E\""; + byte[] bytes = new byte[input.length() << 1]; + StringSerializer.copyStringCharsToBytes(input, bytes); + Utf16JsonReader reader = new Utf16JsonReader().resetUtf16Bytes(input, bytes); + assertEquals(reader.readString(), "music \uD834\uDD1E"); + reader.finish(); + } + + @Test + public void readUtf16FieldNameProbe() { + long hash = JsonFieldNameHash.hash("duration"); + int length = "duration".length(); + long mask = packedNameMask(length); + + Utf16JsonReader direct = utf16Reader("\"duration\":123"); + assertTrue(direct.tryReadNextFieldNameColon(hash, mask, length)); + assertEquals(direct.readNextIntValue(), 123); + direct.finish(); + + Utf16JsonReader mismatch = utf16Reader("\"durable\":7"); + assertFalse(mismatch.tryReadNextFieldNameColon(hash, mask, length)); + assertEquals(mismatch.readFieldNameHash(), JsonFieldNameHash.hash("durable")); + mismatch.expectNextToken(':'); + assertEquals(mismatch.readNextIntValue(), 7); + mismatch.finish(); + + Utf16JsonReader escaped = utf16Reader("\"dur\\u0061tion\":9"); + assertFalse(escaped.tryReadNextFieldNameColon(hash, mask, length)); + assertEquals(escaped.readFieldNameHash(), hash); + escaped.expectNextToken(':'); + assertEquals(escaped.readNextIntValue(), 9); + escaped.finish(); + } + + @Test + public void readUtf16FieldNameTokenProbe() { + String shortToken = "\"size\":"; + int shortTail = shortToken.length() - 4; + Utf16JsonReader shortDirect = utf16Reader("\"size\":7"); + assertTrue( + shortDirect.tryReadNextFieldNameUtf16Token2( + utf16Word(shortToken, 0, 4), + -1L, + utf16Word(shortToken, 4, shortTail), + utf16WordMask(shortTail), + shortToken.length())); + assertEquals(shortDirect.readIntTokenValue(), 7); + shortDirect.finish(); + + String name = "duration"; + String token = "\"" + name + "\":"; + long firstWord = utf16Word(token, 0, 4); + long secondWord = utf16Word(token, 4, 4); + long thirdWord = utf16Word(token, 8, token.length() - 8); + + Utf16JsonReader direct = utf16Reader("\"duration\":123"); + assertTrue( + direct.tryReadNextFieldNameUtf16Token3(firstWord, secondWord, thirdWord, token.length())); + assertEquals(direct.readIntTokenValue(), 123); + direct.finish(); + + Utf16JsonReader nullable = utf16Reader("\"duration\":null"); + assertTrue( + nullable.tryReadNextFieldNameUtf16Token3(firstWord, secondWord, thirdWord, token.length())); + assertEquals(nullable.readNullableStringToken(), null); + nullable.finish(); + + Utf16JsonReader spaced = utf16Reader("\"duration\" : 123"); + assertFalse( + spaced.tryReadNextFieldNameUtf16Token3(firstWord, secondWord, thirdWord, token.length())); + assertTrue( + spaced.tryReadNextFieldNameUtf16( + JsonFieldNameHash.hash(name), + packedNameMask(8), + utf16Word(name, 0, 4), + -1L, + utf16Word(name, 4, 4), + -1L, + 8)); + assertEquals(spaced.readNextIntValue(), 123); + spaced.finish(); + } + @Test public void writeUtf16Char() { ForyJson json = ForyJson.builder().build(); @@ -105,21 +195,46 @@ public void stringWriterResetAfterMaterialize() { String utf16Json = writer.toJson(); writer.reset(); writer.writeString("ascii"); + String latin1Json = writer.toJson(); assertEquals(utf16Json, "\"你好，Fory\""); - assertEquals(writer.toJson(), "\"ascii\""); + assertEquals(latin1Json, "\"ascii\""); if (StringSerializer.isBytesBackedString()) { assertTrue(StringSerializer.isUtf16Coder(StringSerializer.getStringCoder(utf16Json))); + assertTrue(StringSerializer.isLatin1Coder(StringSerializer.getStringCoder(latin1Json))); } } + @Test + public void stringWriterLatin1AfterUtf16() { + StringJsonWriter writer = new StringJsonWriter(false, new byte[16]); + writer.writeArrayStart(); + writer.writeStringElement(0, "你好"); + writer.writeStringElement(1, "http://example.com/keynote.jpg"); + writer.writeStringElement(2, "café"); + writer.writeStringElement(3, "line\nbreak"); + writer.writeArrayEnd(); + assertEquals( + writer.toJson(), "[\"你好\",\"http://example.com/keynote.jpg\",\"café\",\"line\\nbreak\"]"); + } + + @Test + public void stringWriterUtf16Escapes() { + StringJsonWriter writer = new StringJsonWriter(false, new byte[16]); + writer.writeArrayStart(); + writer.writeStringElement(0, "你好"); + writer.writeStringElement(1, "前缀\"\\\\\n\u1234"); + writer.writeArrayEnd(); + assertEquals(writer.toJson(), "[\"你好\",\"前缀\\\"\\\\\\\\\\n\u1234\"]"); + } + @Test public void stringWriterShrinksOnReset() throws Exception { StringJsonWriter writer = new StringJsonWriter(false, new byte[16]); - writer.writeString(repeat('a', 9000) + "你好，Fory"); - assertTrue(writerBufferLength(writer) > 8192); + writer.writeString(repeat('a', 40000) + "你好，Fory"); + assertTrue(writerBufferLength(writer) > 65536); writer.toJson(); writer.reset(); - assertEquals(writerBufferLength(writer), 8192); + assertEquals(writerBufferLength(writer), 65536); writer.writeString("café"); assertEquals(writer.toJson(), "\"café\""); } @@ -139,6 +254,62 @@ public void readStringInputLayouts() { assertEquals(json.fromJson("\"你好，Fory\"", String.class), ZH_TEXT); } + @Test + public void readStringInputEscapes() { + ForyJson json = ForyJson.builder().build(); + + String asciiEscaped = json.fromJson("\"line\\n\\r\\t\\b\\f\\\\\\\"\\/end\"", String.class); + String latin1Escaped = json.fromJson("\"caf\\u00E9\"", String.class); + String chineseEscaped = json.fromJson("\"\\u4E2D\\u6587\"", String.class); + String supplementary = json.fromJson("\"\\uD83D\\uDE00\"", String.class); + + assertEquals(asciiEscaped, "line\n\r\t\b\f\\\"/end"); + assertEquals(latin1Escaped, "café"); + assertEquals(chineseEscaped, "中文"); + assertEquals(supplementary, "\uD83D\uDE00"); + assertLatin1String(asciiEscaped); + assertLatin1String(latin1Escaped); + assertUtf16String(chineseEscaped); + assertUtf16String(supplementary); + + String utf16Escaped = "\"中文\\n\\u00E9\\uD83D\\uDE00\""; + if (StringSerializer.isBytesBackedString()) { + assertTrue(StringSerializer.isUtf16Coder(StringSerializer.getStringCoder(utf16Escaped))); + } + assertEquals(json.fromJson(utf16Escaped, String.class), "中文\né\uD83D\uDE00"); + + String utf16Object = "{\"active\":true,\"id\":7,\"name\":\"line\\n\",\"ignored\":\"中文\"}"; + PublicFields fields = json.fromJson(utf16Object, PublicFields.class); + assertEquals(fields.name, "line\n"); + assertLatin1String(fields.name); + + assertThrows(ForyJsonException.class, () -> json.fromJson("\"bad\\x\"", String.class)); + assertThrows(ForyJsonException.class, () -> json.fromJson("\"中文\\x\"", String.class)); + assertThrows(ForyJsonException.class, () -> json.fromJson("\"bad\u001f\"", String.class)); + assertThrows(ForyJsonException.class, () -> json.fromJson("\"中文\u001f\"", String.class)); + assertThrows(ForyJsonException.class, () -> json.fromJson("\"\\uD83D\"", String.class)); + } + + @Test + public void readerDecodeBufferShrinks() throws Exception { + String latin1Input = "\"" + repeat('a', 9000) + "\\n\""; + if (StringSerializer.isBytesBackedString() + && StringSerializer.isLatin1Coder(StringSerializer.getStringCoder(latin1Input))) { + Latin1JsonReader latin1Reader = new Latin1JsonReader(latin1Input); + assertEquals(latin1Reader.readString(), repeat('a', 9000) + "\n"); + assertTrue(readerBufferLength(latin1Reader) > 8192); + latin1Reader.clear(); + assertEquals(readerBufferLength(latin1Reader), 8192); + } + + String utf16Input = "\"中文" + repeat('b', 9000) + "\\n\""; + Utf16JsonReader utf16Reader = new Utf16JsonReader(utf16Input); + assertEquals(utf16Reader.readString(), "中文" + repeat('b', 9000) + "\n"); + assertTrue(readerBufferLength(utf16Reader) > 8192); + utf16Reader.clear(); + assertEquals(readerBufferLength(utf16Reader), 8192); + } + @Test public void readUnicodeFieldNames() { ForyJson json = ForyJson.builder().build(); @@ -224,15 +395,37 @@ public void writeSurrogatePair() { assertEquals(json.fromJson(stringExpected, PublicFields.class).name, fields.name); } + @Test + public void writeStringScanBoundaries() { + ForyJson json = ForyJson.builder().build(); + for (int length : + new int[] { + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 15, 16, 17, 20, 23, 24, 25, 30, 31, 32, 33, 63, 64, 65 + }) { + String value = repeat('a', length); + String expected = "\"" + value + "\""; + assertEquals(json.toJson(value), expected); + assertEquals(new String(json.toJsonBytes(value), StandardCharsets.UTF_8), expected); + } + String escaped = repeat('b', 24) + "\n"; + String expected = "\"" + repeat('b', 24) + "\\n\""; + assertEquals(json.toJson(escaped), expected); + assertEquals(new String(json.toJsonBytes(escaped), StandardCharsets.UTF_8), expected); + } + @Test public void readStringScanBoundaries() { ForyJson json = ForyJson.builder().build(); - for (int length : new int[] {7, 8, 15, 16, 23, 24}) { + for (int length : new int[] {0, 1, 7, 8, 15, 16, 17, 23, 24, 31, 32, 33, 63, 64, 65}) { String value = repeat('a', length); String input = "\"" + value + "\""; assertEquals(json.fromJson(input, String.class), value); assertEquals(json.fromJson(input.getBytes(StandardCharsets.UTF_8), String.class), value); } + for (int length : new int[] {7, 8, 15, 16, 23, 24, 31, 32}) { + String value = repeat('a', length) + "\u00e9" + repeat('b', 3); + assertEquals(json.fromJson("\"" + value + "\"", String.class), value); + } String escapedValue = repeat('b', 16) + "\n"; String escapedInput = "\"" + repeat('b', 16) + "\\n\""; assertEquals(json.fromJson(escapedInput, String.class), escapedValue); @@ -253,6 +446,41 @@ public void readStringScanBoundaries() { assertThrows(ForyJsonException.class, () -> json.fromJson(invalidUtf8, String.class)); } + @Test + public void readUtf8DecodedStrings() { + ForyJson json = ForyJson.builder().build(); + String ascii = readUtf8String(json, "\"plain ascii\""); + String latin1 = readUtf8String(json, "\"caf\u00e9\""); + String escaped = readUtf8String(json, "\"line\\n\\t\\\\\\\"\\/end\""); + String escapedLatin1 = readUtf8String(json, "\"caf\\u00e9\""); + String chinese = readUtf8String(json, "\"\u4e2d\u6587\""); + String escapedChinese = readUtf8String(json, "\"\\u4e2d\\u6587\""); + String supplementary = readUtf8String(json, "\"\\uD83D\\uDE00\""); + + assertEquals(ascii, "plain ascii"); + assertEquals(latin1, "caf\u00e9"); + assertEquals(escaped, "line\n\t\\\"/end"); + assertEquals(escapedLatin1, "caf\u00e9"); + assertEquals(chinese, "\u4e2d\u6587"); + assertEquals(escapedChinese, "\u4e2d\u6587"); + assertEquals(supplementary, "\uD83D\uDE00"); + assertLatin1String(ascii); + assertLatin1String(latin1); + assertLatin1String(escaped); + assertLatin1String(escapedLatin1); + assertUtf16String(chinese); + assertUtf16String(escapedChinese); + assertUtf16String(supplementary); + + byte[] rawControl = {'"', 'b', 'a', 'd', 0x1f, '"'}; + byte[] malformedUtf8 = {'"', (byte) 0xE4, (byte) 0xB8, '"'}; + assertThrows(ForyJsonException.class, () -> json.fromJson(rawControl, String.class)); + assertThrows(ForyJsonException.class, () -> json.fromJson(malformedUtf8, String.class)); + assertThrows( + ForyJsonException.class, + () -> json.fromJson("\"\\uD800\"".getBytes(StandardCharsets.UTF_8), String.class)); + } + @Test public void rejectInvalidSurrogates() { ForyJson json = ForyJson.builder().build(); @@ -269,4 +497,54 @@ private static int writerBufferLength(StringJsonWriter writer) throws Exception field.setAccessible(true); return ((byte[]) field.get(writer)).length; } + + private static int readerBufferLength(Object reader) throws Exception { + Field field = reader.getClass().getDeclaredField("stringDecodeBuffer"); + field.setAccessible(true); + return ((byte[]) field.get(reader)).length; + } + + private static String readUtf8String(ForyJson json, String input) { + return json.fromJson(input.getBytes(StandardCharsets.UTF_8), String.class); + } + + private static void assertLatin1String(String value) { + if (StringSerializer.isBytesBackedString()) { + assertTrue(StringSerializer.isLatin1Coder(StringSerializer.getStringCoder(value))); + } + } + + private static void assertUtf16String(String value) { + if (StringSerializer.isBytesBackedString()) { + assertTrue(StringSerializer.isUtf16Coder(StringSerializer.getStringCoder(value))); + } + } + + private static Utf16JsonReader utf16Reader(String input) { + byte[] bytes = new byte[input.length() << 1]; + StringSerializer.copyStringCharsToBytes(input, bytes); + return new Utf16JsonReader().resetUtf16Bytes(input, bytes); + } + + private static long packedNameMask(int length) { + return length == Long.BYTES ? -1L : (1L << (length << 3)) - 1L; + } + + private static long utf16WordMask(int length) { + return length == 4 ? -1L : (1L << (length << 4)) - 1L; + } + + private static long utf16Word(String value, int start, int length) { + long packed = 0; + for (int i = 0; i < length; i++) { + packed |= (long) (value.charAt(start + i) & 0xFF) << (i << 3); + } + long word = spreadLatin1ToUtf16(packed); + return NativeByteOrder.IS_LITTLE_ENDIAN ? word : word << 8; + } + + private static long spreadLatin1ToUtf16(long value) { + value = (value | (value << 16)) & 0x0000FFFF0000FFFFL; + return (value | (value << 8)) & 0x00FF00FF00FF00FFL; + } } diff --git a/java/fory-json/src/test/java/org/apache/fory/json/data/BeanProperties.java b/java/fory-json/src/test/java/org/apache/fory/json/data/BeanProperties.java new file mode 100644 index 0000000000..b54169e461 --- /dev/null +++ b/java/fory-json/src/test/java/org/apache/fory/json/data/BeanProperties.java @@ -0,0 +1,186 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.json.data; + +public final class BeanProperties { + private BeanProperties() {} + + public static final class GetterBean { + private int id = 7; + private String name = "field"; + + public int getId() { + return id + 10; + } + + public String getName() { + return "getter-" + name; + } + } + + public static final class SetterBean { + private int id; + private String name; + private int setterCalls; + + public void setId(int id) { + this.id = id + 1; + setterCalls++; + } + + public void setName(String name) { + this.name = "set-" + name; + setterCalls++; + } + + public static int id(SetterBean value) { + return value.id; + } + + public static String name(SetterBean value) { + return value.name; + } + + public static int setterCalls(SetterBean value) { + return value.setterCalls; + } + } + + public static final class MixedBean { + public int count = 3; + private String name = "mixed"; + private int score = 5; + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public int getScore() { + return score; + } + + public void setScore(int score) { + this.score = score; + } + } + + public static final class BooleanBean { + private boolean ready = true; + + public boolean isReady() { + return ready; + } + } + + public static final class GetterOnlyBean { + private transient int seed = 3; + + public int getComputed() { + return seed * 2; + } + } + + public static final class SetterOnlyBean { + private transient String received; + + public void setSecret(String secret) { + received = "set-" + secret; + } + + public static String received(SetterOnlyBean value) { + return value.received; + } + } + + public static final class DuplicateGetterBean { + public boolean getActive() { + return true; + } + + public boolean isActive() { + return true; + } + } + + public static final class OverloadedSetterBean { + public void setValue(int value) {} + + public void setValue(String value) {} + } + + public static final class ConflictingTypesBean { + private String value = "text"; + + public String getValue() { + return value; + } + + public void setValue(int value) {} + } + + public static class InheritedParent { + private int id = 4; + + public int getId() { + return id; + } + + public void setId(int id) { + this.id = id + 1; + } + + public static int id(InheritedParent value) { + return value.id; + } + } + + public static final class InheritedChild extends InheritedParent { + public String name = "child"; + } + + public static final class InvalidAccessorBean { + public String value = "field"; + + public static String getStaticValue() { + return "static"; + } + + public String getWithArg(String ignored) { + return ignored; + } + + public void getVoidValue() {} + + public String setWrongReturn(String value) { + return value; + } + + public void setNoValue() {} + } + + public static final class FinalFieldBean { + public final int id = 1; + public String name = "field"; + } +}