From 41662c78459976dc01fb32fbea41aae56a48d99c Mon Sep 17 00:00:00 2001 From: Shaojin Wen Date: Wed, 15 Jan 2025 23:43:32 +0800 Subject: [PATCH] refactor (#3282) * refactor * add testcase * refactor * checkstyle * refactor --- .../fastjson2/benchmark/CSVBankTest.java | 8 +- .../fastjson2/JSONWriterUTF16JDK9UF.java | 8 +- .../com/alibaba/fastjson2/JSONWriterUTF8.java | 8 +- .../fastjson2/support/csv/CSVReaderUTF16.java | 20 +- .../fastjson2/support/csv/CSVReaderUTF8.java | 36 +- .../fastjson2/support/csv/CSVWriterUTF16.java | 26 +- .../fastjson2/support/csv/CSVWriterUTF8.java | 29 +- .../com/alibaba/fastjson2/util/IOUtils.java | 325 ++++++++++-------- .../fastjson2/support/csv/CSVWriterTest.java | 20 ++ 9 files changed, 249 insertions(+), 231 deletions(-) diff --git a/benchmark/src/test/java/com/alibaba/fastjson2/benchmark/CSVBankTest.java b/benchmark/src/test/java/com/alibaba/fastjson2/benchmark/CSVBankTest.java index 3c81d65f61..b3e2c3dade 100644 --- a/benchmark/src/test/java/com/alibaba/fastjson2/benchmark/CSVBankTest.java +++ b/benchmark/src/test/java/com/alibaba/fastjson2/benchmark/CSVBankTest.java @@ -4,7 +4,7 @@ public class CSVBankTest { static final CSVBank benchmark = new CSVBank(); - static final int LOOP = 1000; + static final int LOOP = 10000; public static void fastjson2() { for (int j = 0; j < 5; j++) { @@ -16,7 +16,7 @@ public static void fastjson2() { System.out.println("fastjson2 millis : " + millis); // zulu8.68.0.21 : 213 // zulu11.62.17 : 148 - // zulu17.40.19 : 150 + // zulu17.40.19 : 150 1746 1639 } } @@ -49,8 +49,8 @@ public static void cainiao() throws Exception { } public static void main(String[] args) throws Exception { -// fastjson2(); - univocity(); + fastjson2(); +// univocity(); // cainiao(); } } diff --git a/core/src/main/java/com/alibaba/fastjson2/JSONWriterUTF16JDK9UF.java b/core/src/main/java/com/alibaba/fastjson2/JSONWriterUTF16JDK9UF.java index 787c6cdbd3..52c4ebc8f6 100644 --- a/core/src/main/java/com/alibaba/fastjson2/JSONWriterUTF16JDK9UF.java +++ b/core/src/main/java/com/alibaba/fastjson2/JSONWriterUTF16JDK9UF.java @@ -78,13 +78,7 @@ public void writeBool(boolean value) { if ((context.features & WriteBooleanAsNumber.mask) != 0) { chars[off++] = value ? '1' : '0'; } else { - if (value) { - IOUtils.putTrue(chars, off); - off += 4; - } else { - IOUtils.putFalse(chars, off); - off += 5; - } + off = IOUtils.putBoolean(chars, off, value); } this.off = off; } diff --git a/core/src/main/java/com/alibaba/fastjson2/JSONWriterUTF8.java b/core/src/main/java/com/alibaba/fastjson2/JSONWriterUTF8.java index b7605560c0..d6c85fa2ff 100644 --- a/core/src/main/java/com/alibaba/fastjson2/JSONWriterUTF8.java +++ b/core/src/main/java/com/alibaba/fastjson2/JSONWriterUTF8.java @@ -2927,13 +2927,7 @@ public void writeBool(boolean value) { if ((context.features & WriteBooleanAsNumber.mask) != 0) { bytes[off++] = (byte) (value ? '1' : '0'); } else { - if (value) { - IOUtils.putTrue(bytes, off); - off += 4; - } else { - IOUtils.putFalse(bytes, off); - off += 5; - } + off = IOUtils.putBoolean(bytes, off, value); } this.off = off; } diff --git a/core/src/main/java/com/alibaba/fastjson2/support/csv/CSVReaderUTF16.java b/core/src/main/java/com/alibaba/fastjson2/support/csv/CSVReaderUTF16.java index 0fd3a8bf86..f52f42509d 100644 --- a/core/src/main/java/com/alibaba/fastjson2/support/csv/CSVReaderUTF16.java +++ b/core/src/main/java/com/alibaba/fastjson2/support/csv/CSVReaderUTF16.java @@ -21,6 +21,7 @@ import java.util.function.Consumer; import java.util.function.Function; +import static com.alibaba.fastjson2.support.csv.CSVReaderUTF8.containsQuoteOrLineSeparator; import static com.alibaba.fastjson2.util.DateUtils.DEFAULT_ZONE_ID; final class CSVReaderUTF16 @@ -97,19 +98,11 @@ protected boolean seekLine() throws IOException { for (int k = 0; k < 3; ++k) { lineTerminated = false; - for (int i = off; i < end; i++) { - if (i + 4 < end) { - char b0 = buf[i]; - char b1 = buf[i + 1]; - char b2 = buf[i + 2]; - char b3 = buf[i + 3]; - if (b0 > '"' && b1 > '"' && b2 > '"' && b3 > '"') { - lineSize += 4; - i += 3; - continue; - } - } - + int i = off, end = this.end; + while (i + 4 < end && !containsQuoteOrLineSeparator(IOUtils.getLongUnaligned(buf, i))) { + i += 4; + } + for (; i < end; i++) { char ch = buf[i]; if (ch == '"') { lineSize++; @@ -194,6 +187,7 @@ protected boolean seekLine() throws IOException { } } else { end += cnt; + this.end = end; continue; } } diff --git a/core/src/main/java/com/alibaba/fastjson2/support/csv/CSVReaderUTF8.java b/core/src/main/java/com/alibaba/fastjson2/support/csv/CSVReaderUTF8.java index 6e14fe28a9..142b99c932 100644 --- a/core/src/main/java/com/alibaba/fastjson2/support/csv/CSVReaderUTF8.java +++ b/core/src/main/java/com/alibaba/fastjson2/support/csv/CSVReaderUTF8.java @@ -87,6 +87,26 @@ final class CSVReaderUTF8 this.valueConsumer = valueConsumer; } + static boolean containsQuoteOrLineSeparator(long v) { + /* + for (int i = 0; i < 8; ++i) { + byte c = (byte) v; + if (c == '"' || c == '\n' || c == '\r') { + return true; + } + v >>>= 8; + } + return false; + */ + long x22 = v ^ 0x2222222222222222L; // " -> 0x22 + long x0a = v ^ 0x0A0A0A0A0A0A0A0AL; // \n -> 0x0a + long x0d = v ^ 0x0D0D0D0D0D0D0D0DL; // \r -> 0x0d + x22 = (x22 - 0x0101010101010101L) & ~x22; + x0a = (x0a - 0x0101010101010101L) & ~x0a; + x0d = (x0d - 0x0101010101010101L) & ~x0d; + return ((x22 | x0a | x0d) & 0x8080808080808080L) != 0; + } + protected boolean seekLine() throws IOException { byte[] buf = this.buf; int off = this.off; @@ -100,12 +120,9 @@ protected boolean seekLine() throws IOException { } this.end = cnt; - if (end > 3) { - // UTF8-BOM EF BB BF - if (buf[0] == -17 && buf[1] == -69 && buf[2] == -65) { - off = 3; - lineNextStart = off; - } + if (end > 4 && IOUtils.isUTF8BOM(buf, 0)) { + off = 3; + lineNextStart = off; } } } @@ -113,7 +130,11 @@ protected boolean seekLine() throws IOException { for (int k = 0; k < 3; ++k) { lineTerminated = false; - for (int i = off; i < end; i++) { + int i = off, end = this.end; + while (i + 8 < end && !containsQuoteOrLineSeparator(IOUtils.getLongUnaligned(buf, i))) { + i += 8; + } + for (; i < end; i++) { byte ch = buf[i]; if (ch == '"') { lineSize++; @@ -198,6 +219,7 @@ protected boolean seekLine() throws IOException { } } else { end += cnt; + this.end = end; continue; } } diff --git a/core/src/main/java/com/alibaba/fastjson2/support/csv/CSVWriterUTF16.java b/core/src/main/java/com/alibaba/fastjson2/support/csv/CSVWriterUTF16.java index 1ae662895a..ba69415639 100644 --- a/core/src/main/java/com/alibaba/fastjson2/support/csv/CSVWriterUTF16.java +++ b/core/src/main/java/com/alibaba/fastjson2/support/csv/CSVWriterUTF16.java @@ -12,12 +12,11 @@ import java.time.LocalDateTime; import java.time.ZoneId; -import static com.alibaba.fastjson2.util.IOUtils.*; - final class CSVWriterUTF16 extends CSVWriter { final Writer out; final char[] chars; + private static final int DOUBLE_QUOTE_2_UTF16 = '"' | ('"' << 16); CSVWriterUTF16( Writer out, @@ -52,17 +51,9 @@ public void writeLine() { chars[off++] = '\n'; } - public void writeBoolean(boolean booleanValue) { - int size = booleanValue ? 4 : 5; - checkCapacity(size); - char[] chars = this.chars; - int off = this.off; - if (booleanValue) { - IOUtils.putTrue(chars, off); - } else { - IOUtils.putFalse(chars, off); - } - this.off = off + size; + public void writeBoolean(boolean v) { + checkCapacity(5); + this.off = IOUtils.putBoolean(chars, off, v); } public void writeInt64(long longValue) { @@ -88,11 +79,7 @@ public void writeDateTime19( int off = this.off; off = IOUtils.writeLocalDate(chars, off, year, month, dayOfMonth); chars[off] = ' '; - writeDigitPair(chars, off + 1, hour); - chars[off + 3] = ':'; - writeDigitPair(chars, off + 4, minute); - chars[off + 6] = ':'; - writeDigitPair(chars, off + 7, second); + IOUtils.writeLocalTime(chars, off + 1, hour, minute, second); this.off = off + 9; } @@ -153,8 +140,7 @@ public void writeString(final String str) { for (int i = 0; i < len; ) { char ch = str.charAt(i++); if (ch == '"') { - chars[off] = '"'; - chars[off + 1] = '"'; + IOUtils.putIntUnaligned(chars, off, DOUBLE_QUOTE_2_UTF16); off += 2; } else { chars[off++] = ch; diff --git a/core/src/main/java/com/alibaba/fastjson2/support/csv/CSVWriterUTF8.java b/core/src/main/java/com/alibaba/fastjson2/support/csv/CSVWriterUTF8.java index 306f7f77ac..7559fc016e 100644 --- a/core/src/main/java/com/alibaba/fastjson2/support/csv/CSVWriterUTF8.java +++ b/core/src/main/java/com/alibaba/fastjson2/support/csv/CSVWriterUTF8.java @@ -12,10 +12,10 @@ import java.time.LocalDateTime; import java.time.ZoneId; -import static com.alibaba.fastjson2.util.IOUtils.*; - final class CSVWriterUTF8 extends CSVWriter { + private static final short DOUBLE_QUOTE_2_LATIN1 = 0x22 | (0x22 << 8); + final OutputStream out; final Charset charset; final byte[] bytes; @@ -55,28 +55,18 @@ public void writeLine() { bytes[off++] = '\n'; } - public void writeBoolean(boolean booleanValue) { - int size = booleanValue ? 4 : 5; - checkCapacity(size); - byte[] chars = this.bytes; - int off = this.off; - if (booleanValue) { - IOUtils.putTrue(chars, off); - } else { - IOUtils.putFalse(chars, off); - } - this.off = off + size; + public void writeBoolean(boolean v) { + checkCapacity(5); + this.off = IOUtils.putBoolean(this.bytes, off, v); } public void writeInt64(long longValue) { checkCapacity(20); // -9223372036854775808 - off = IOUtils.writeInt64(bytes, off, longValue); } public void writeDateYYYMMDD10(int year, int month, int dayOfMonth) { checkCapacity(10); - off = IOUtils.writeLocalDate(bytes, off, year, month, dayOfMonth); } @@ -94,11 +84,7 @@ public void writeDateTime19( int off = this.off; off = IOUtils.writeLocalDate(bytes, off, year, month, dayOfMonth); bytes[off] = ' '; - writeDigitPair(bytes, off + 1, hour); - bytes[off + 3] = ':'; - writeDigitPair(bytes, off + 4, minute); - bytes[off + 6] = ':'; - writeDigitPair(bytes, off + 7, second); + IOUtils.writeLocalTime(bytes, off + 1, hour, minute, second); this.off = off + 9; } @@ -187,8 +173,7 @@ public void writeString(byte[] utf8) { bytes[off++] = '"'; for (byte ch : utf8) { if (ch == '"') { - bytes[off] = '"'; - bytes[off + 1] = '"'; + IOUtils.putShortUnaligned(bytes, off, DOUBLE_QUOTE_2_LATIN1); off += 2; } else { bytes[off++] = ch; diff --git a/core/src/main/java/com/alibaba/fastjson2/util/IOUtils.java b/core/src/main/java/com/alibaba/fastjson2/util/IOUtils.java index 6d79c8354c..416a541a39 100644 --- a/core/src/main/java/com/alibaba/fastjson2/util/IOUtils.java +++ b/core/src/main/java/com/alibaba/fastjson2/util/IOUtils.java @@ -34,7 +34,8 @@ public class IOUtils { private static final byte[] MIN_INT_BYTES = "-2147483648".getBytes(); private static final char[] MIN_INT_CHARS = "-2147483648".toCharArray(); - private static final byte[] MIN_LONG = "-9223372036854775808".getBytes(); + private static final byte[] MIN_LONG_BYTES = "-9223372036854775808".getBytes(); + private static final char[] MIN_LONG_CHARS = "-9223372036854775808".toCharArray(); public static final short[] PACKED_DIGITS; public static final int[] PACKED_DIGITS_UTF16; @@ -60,6 +61,9 @@ public class IOUtils { 1000000000000000000L, }; + private static final short ZERO_DOT_LATIN1; + private static final int ZERO_DOT_UTF16; + static { short[] shorts = new short[]{ 0x3030, 0x3130, 0x3230, 0x3330, 0x3430, 0x3530, 0x3630, 0x3730, 0x3830, 0x3930, @@ -113,6 +117,8 @@ public class IOUtils { } DIGITS_K_64[i] = c0 + v; } + ZERO_DOT_LATIN1 = UNSAFE.getShort(new byte[] {'0', '.'}, ARRAY_BYTE_BASE_OFFSET); + ZERO_DOT_UTF16 = UNSAFE.getInt(new char[] {'0', '.'}, ARRAY_BYTE_BASE_OFFSET); } public static void writeDigitPair(byte[] buf, int charPos, int value) { @@ -171,11 +177,11 @@ public static void getChars(int i, int index, byte[] buf) { charPos -= 2; writeDigitPair(buf, charPos, -i); } else { - buf[--charPos] = (byte) ('0' - i); + putByte(buf, --charPos, (byte) ('0' - i)); } if (negative) { - buf[charPos - 1] = (byte) '-'; + putByte(buf, charPos - 1, (byte) '-'); } } @@ -203,11 +209,11 @@ public static void getChars(int i, int index, char[] buf) { charPos -= 2; writeDigitPair(buf, charPos, -i); } else { - buf[--charPos] = (char) ('0' - i); + putChar(buf, --charPos, (char) ('0' - i)); } if (negative) { - buf[charPos - 1] = '-'; + putChar(buf, charPos - 1, '-'); } } @@ -244,11 +250,11 @@ public static void getChars(long i, int index, byte[] buf) { charPos -= 2; writeDigitPair(buf, charPos, -i2); } else { - buf[--charPos] = (byte) ('0' - i2); + putByte(buf, --charPos, (byte) ('0' - i2)); } if (negative) { - buf[charPos - 1] = (byte) '-'; + putByte(buf, charPos - 1, (byte) '-'); } } @@ -284,17 +290,17 @@ public static void getChars(long i, int index, char[] buf) { charPos -= 2; writeDigitPair(buf, charPos, -i2); } else { - buf[--charPos] = (char) ('0' - i2); + putChar(buf, --charPos, (char) ('0' - i2)); } if (negative) { - buf[--charPos] = '-'; + putChar(buf, charPos - 1, '-'); } } public static int writeDecimal(byte[] buf, int off, long unscaledVal, int scale) { if (unscaledVal < 0) { - buf[off++] = (byte) '-'; + putByte(buf, off++, (byte) '-'); unscaledVal = -unscaledVal; } @@ -302,26 +308,24 @@ public static int writeDecimal(byte[] buf, int off, long unscaledVal, int scale) int unscaleValSize = IOUtils.stringSize(unscaledVal); int insertionPoint = unscaleValSize - scale; if (insertionPoint == 0) { - buf[off] = '0'; - buf[off + 1] = '.'; + putShortUnaligned(buf, off, ZERO_DOT_LATIN1); off += 2; } else if (insertionPoint < 0) { - buf[off] = '0'; - buf[off + 1] = '.'; + putShortUnaligned(buf, off, ZERO_DOT_LATIN1); off += 2; for (int i = 0; i < -insertionPoint; i++) { - buf[off++] = '0'; + putByte(buf, off++, (byte) '0'); } } else { long power = POWER_TEN[scale - 1]; long div = unscaledVal / power; long rem = unscaledVal - div * power; off = IOUtils.writeInt64(buf, off, div); - buf[off] = '.'; + putByte(buf, off, (byte) '.'); if (scale == 1) { - buf[off + 1] = (byte) (rem + '0'); + putByte(buf, off + 1, (byte) (rem + '0')); return off + 2; } else if (scale == 2) { writeDigitPair(buf, off + 1, (int) rem); @@ -329,7 +333,7 @@ public static int writeDecimal(byte[] buf, int off, long unscaledVal, int scale) } for (int i = 0, end = unscaleValSize - stringSize(rem) - insertionPoint; i < end; ++i) { - buf[++off] = '0'; + putByte(buf, ++off, (byte) '0'); } return IOUtils.writeInt64(buf, off + 1, rem); } @@ -340,7 +344,7 @@ public static int writeDecimal(byte[] buf, int off, long unscaledVal, int scale) public static int writeDecimal(char[] buf, int off, long unscaledVal, int scale) { if (unscaledVal < 0) { - buf[off++] = (byte) '-'; + putChar(buf, off++, '-'); unscaledVal = -unscaledVal; } @@ -348,26 +352,24 @@ public static int writeDecimal(char[] buf, int off, long unscaledVal, int scale) int unscaleValSize = stringSize(unscaledVal); int insertionPoint = unscaleValSize - scale; if (insertionPoint == 0) { - buf[off] = '0'; - buf[off + 1] = '.'; + putIntUnaligned(buf, off, ZERO_DOT_UTF16); off += 2; } else if (insertionPoint < 0) { - buf[off] = '0'; - buf[off + 1] = '.'; + putIntUnaligned(buf, off, ZERO_DOT_UTF16); off += 2; for (int i = 0; i < -insertionPoint; i++) { - buf[off++] = '0'; + putChar(buf, off++, '0'); } } else { long power = POWER_TEN[scale - 1]; long div = unscaledVal / power; long rem = unscaledVal - div * power; off = IOUtils.writeInt64(buf, off, div); - buf[off] = '.'; + putChar(buf, off, '.'); if (scale == 1) { - buf[off + 1] = (char) (rem + '0'); + putChar(buf, off + 1, (char) (rem + '0')); return off + 2; } else if (scale == 2) { writeDigitPair(buf, off + 1, (int) rem); @@ -375,7 +377,7 @@ public static int writeDecimal(char[] buf, int off, long unscaledVal, int scale) } for (int i = 0, end = unscaleValSize - stringSize(rem) - insertionPoint; i < end; ++i) { - buf[++off] = '0'; + putChar(buf, ++off, '0'); } return IOUtils.writeInt64(buf, off + 1, rem); } @@ -753,10 +755,10 @@ public static long lines(InputStream in) throws Exception { public static int writeLocalDate(byte[] bytes, int off, int year, int month, int dayOfMonth) { if (year < 0) { - bytes[off++] = '-'; + putByte(bytes, off++, (byte) '-'); year = -year; } else if (year > 9999) { - bytes[off++] = '+'; + putByte(bytes, off++, (byte) '+'); } if (year < 10000) { @@ -769,19 +771,19 @@ public static int writeLocalDate(byte[] bytes, int off, int year, int month, int off = IOUtils.writeInt32(bytes, off, year); } - bytes[off] = '-'; + putByte(bytes, off, (byte) '-'); writeDigitPair(bytes, off + 1, month); - bytes[off + 3] = '-'; + putByte(bytes, off + 3, (byte) '-'); writeDigitPair(bytes, off + 4, dayOfMonth); return off + 6; } public static int writeLocalDate(char[] chars, int off, int year, int month, int dayOfMonth) { if (year < 0) { - chars[off++] = '-'; + putChar(chars, off++, '-'); year = -year; } else if (year > 9999) { - chars[off++] = '+'; + putChar(chars, off++, '+'); } if (year < 10000) { @@ -794,18 +796,18 @@ public static int writeLocalDate(char[] chars, int off, int year, int month, int off = IOUtils.writeInt32(chars, off, year); } - chars[off] = '-'; + putChar(chars, off, '-'); writeDigitPair(chars, off + 1, month); - chars[off + 3] = '-'; + putChar(chars, off + 3, '-'); writeDigitPair(chars, off + 4, dayOfMonth); return off + 6; } public static void writeLocalTime(byte[] bytes, int off, int hour, int minute, int second) { writeDigitPair(bytes, off, hour); - bytes[off + 2] = ':'; + putByte(bytes, off + 2, (byte) ':'); writeDigitPair(bytes, off + 3, minute); - bytes[off + 5] = ':'; + putByte(bytes, off + 5, (byte) ':'); writeDigitPair(bytes, off + 6, second); } @@ -837,11 +839,10 @@ public static int writeNano(byte[] bytes, int off, int nano) { v = DIGITS_K_32[(div - div2 * 1000) & 0x3ff]; } - bytes[off] = (byte) (v >> 8); - bytes[off + 1] = (byte) (v >> 16); + putShortLE(bytes, off, (short) (v >> 8)); off += 2; if (rem1 == 0) { - bytes[off] = (byte) (v >> 24); + putByte(bytes, off, (byte) (v >> 24)); return off + 1; } @@ -869,11 +870,10 @@ public static int writeNano(char[] chars, int off, int nano) { v = DIGITS_K_64[(div - div2 * 1000) & 0x3ff]; } - chars[off] = (char) (v >> 16); - chars[off + 1] = (char) (v >> 32); + putIntLE(chars, off, (int) (v >> 16)); off += 2; if (rem1 == 0) { - chars[off] = (char) (v >> 48); + putChar(chars, off, (char) (v >> 48)); return off + 1; } @@ -883,9 +883,9 @@ public static int writeNano(char[] chars, int off, int nano) { public static void writeLocalTime(char[] chars, int off, int hour, int minute, int second) { writeDigitPair(chars, off, hour); - chars[off + 2] = ':'; + putChar(chars, off + 2, ':'); writeDigitPair(chars, off + 3, minute); - chars[off + 5] = ':'; + putChar(chars, off + 5, ':'); writeDigitPair(chars, off + 6, second); } @@ -897,15 +897,15 @@ public static int writeLocalTime(char[] chars, int off, LocalTime time) { return nano != 0 ? writeNano(chars, off, nano) : off; } - public static int writeInt64(final byte[] buf, int pos, final long value) { + public static int writeInt64(byte[] buf, int pos, final long value) { long i; if (value < 0) { if (value == Long.MIN_VALUE) { - System.arraycopy(MIN_LONG, 0, buf, pos, MIN_LONG.length); - return pos + MIN_LONG.length; + System.arraycopy(MIN_LONG_BYTES, 0, buf, pos, MIN_LONG_BYTES.length); + return pos + MIN_LONG_BYTES.length; } i = -value; - buf[pos++] = '-'; + putByte(buf, pos++, (byte) ('-')); } else { i = value; } @@ -914,13 +914,12 @@ public static int writeInt64(final byte[] buf, int pos, final long value) { int v = DIGITS_K_32[(int) i & 0x3ff]; int start = v & 0xff; if (start == 0) { - buf[pos] = (byte) (v >> 8); - buf[pos + 1] = (byte) (v >> 16); + putShortLE(buf, pos, (short) (v >> 8)); pos += 2; } else if (start == 1) { - buf[pos++] = (byte) (v >> 16); + putByte(buf, pos++, (byte) (v >> 16)); } - buf[pos++] = (byte) (v >> 24); + putByte(buf, pos++, (byte) (v >> 24)); return pos; } @@ -932,11 +931,10 @@ public static int writeInt64(final byte[] buf, int pos, final long value) { int start = v2 & 0xff; if (start == 0) { - buf[pos] = (byte) (v2 >> 8); - buf[pos + 1] = (byte) (v2 >> 16); + putShortLE(buf, pos, (short) (v2 >> 8)); pos += 2; } else if (start == 1) { - buf[pos++] = (byte) (v2 >> 16); + putByte(buf, pos++, (byte) (v2 >> 16)); } putIntLE(buf, pos, v1 & 0xffffff00 | (v2 >> 24)); return pos + 4; @@ -950,15 +948,13 @@ public static int writeInt64(final byte[] buf, int pos, final long value) { final int v3 = DIGITS_K_32[(int) q2 & 0x3ff]; int start = v3 & 0xff; if (start == 0) { - buf[pos] = (byte) (v3 >> 8); - buf[pos + 1] = (byte) (v3 >> 16); + putShortLE(buf, pos, (short) (v3 >> 8)); pos += 2; } else if (start == 1) { - buf[pos++] = (byte) (v3 >> 16); + putByte(buf, pos++, (byte) (v3 >> 16)); } - buf[pos] = (byte) (v3 >> 24); - buf[pos + 1] = (byte) (v2 >> 8); - buf[pos + 2] = (byte) (v2 >> 16); + putByte(buf, pos, (byte) (v3 >> 24)); + putShortLE(buf, pos + 1, (short) (v2 >> 8)); putIntLE(buf, pos + 3, v1 & 0xffffff00 | (v2 >> 24)); return pos + 7; } @@ -969,14 +965,13 @@ public static int writeInt64(final byte[] buf, int pos, final long value) { final int v4 = DIGITS_K_32[(int) q3 & 0x3ff]; final int start = v4 & 0xff; if (start == 0) { - buf[pos] = (byte) (v4 >> 8); - buf[pos + 1] = (byte) (v4 >> 16); + putShortLE(buf, pos, (short) (v4 >> 8)); pos += 2; } else if (start == 1) { - buf[pos++] = (byte) (v4 >> 16); + putByte(buf, pos++, (byte) (v4 >> 16)); } - buf[pos] = (byte) (v4 >> 24); - buf[pos + 1] = (byte) (v3 >> 8); + putByte(buf, pos, (byte) (v4 >> 24)); + putByte(buf, pos + 1, (byte) (v3 >> 8)); putIntLE(buf, pos + 2, ((v2 & 0x00ffff00) << 8) | (v3 >> 16)); putIntLE(buf, pos + 6, v1 & 0xffffff00 | (v2 >> 24)); return pos + 10; @@ -989,14 +984,13 @@ public static int writeInt64(final byte[] buf, int pos, final long value) { final int v5 = DIGITS_K_32[q4 & 0x3ff]; int start = v5 & 0xff; if (start == 0) { - buf[pos] = (byte) (v5 >> 8); - buf[pos + 1] = (byte) (v5 >> 16); + putShortLE(buf, pos, (short) (v5 >> 8)); pos += 2; } else if (start == 1) { - buf[pos++] = (byte) (v5 >> 16); + putByte(buf, pos++, (byte) (v5 >> 16)); } putIntLE(buf, pos, v4 & 0xffffff00 | (v5 >> 24)); - buf[pos + 4] = (byte) (v3 >> 8); + putByte(buf, pos + 4, (byte) (v3 >> 8)); putIntLE(buf, pos + 5, ((v2 & 0x00ffff00) << 8) | (v3 >> 16)); putIntLE(buf, pos + 9, v1 & 0xffffff00 | (v2 >> 24)); return pos + 13; @@ -1008,38 +1002,36 @@ public static int writeInt64(final byte[] buf, int pos, final long value) { int v = DIGITS_K_32[q5 & 0x3ff]; final int start = v & 0xff; if (start == 0) { - buf[pos] = (byte) (v >> 8); - buf[pos + 1] = (byte) (v >> 16); + putShortLE(buf, pos, (short) (v >> 8)); pos += 2; } else if (start == 1) { - buf[pos++] = (byte) (v >> 16); + putByte(buf, pos++, (byte) (v >> 16)); } - buf[pos++] = (byte) (v >> 24); + putByte(buf, pos++, (byte) (v >> 24)); } else { putIntLE(buf, pos, DIGITS_K_32[(q5 - q6 * 1000) & 0x3ff] & 0xffffff00 | (q6 + '0')); pos += 4; } - buf[pos] = (byte) (v5 >> 8); + putByte(buf, pos, (byte) (v5 >> 8)); putIntLE(buf, pos + 1, ((v4 & 0x00ffff00) << 8) | (v5 >> 16)); putIntLE(buf, pos + 5, v3 & 0xffffff00 | (v4 >> 24)); - buf[pos + 9] = (byte) (v2 >> 8); - buf[pos + 10] = (byte) (v2 >> 16); + putShortLE(buf, pos + 9, (short) (v2 >> 8)); putIntLE(buf, pos + 11, v1 & 0xffffff00 | (v2 >> 24)); return pos + 15; } - public static int writeInt64(final char[] buf, int pos, final long value) { + public static int writeInt64(char[] buf, int pos, final long value) { long i; if (value < 0) { if (value == Long.MIN_VALUE) { - for (int x = 0; x < MIN_LONG.length; x++) { - buf[pos + x] = (char) MIN_LONG[x]; + if (value == Long.MIN_VALUE) { + System.arraycopy(MIN_LONG_CHARS, 0, buf, pos, MIN_LONG_CHARS.length); + return pos + MIN_LONG_CHARS.length; } - return pos + MIN_LONG.length; } i = -value; - buf[pos++] = '-'; + putChar(buf, pos++, '-'); } else { i = value; } @@ -1051,9 +1043,9 @@ public static int writeInt64(final char[] buf, int pos, final long value) { putIntLE(buf, pos, (int) (v >> 16)); pos += 2; } else if (start == 1) { - buf[pos++] = (char) (v >> 32); + putChar(buf, pos++, (char) (v >> 32)); } - buf[pos++] = (char) (v >> 48); + putChar(buf, pos++, (char) (v >> 48)); return pos; } @@ -1067,7 +1059,7 @@ public static int writeInt64(final char[] buf, int pos, final long value) { putIntLE(buf, pos, (int) (v2 >> 16)); pos += 2; } else if (start == 1) { - buf[pos++] = (char) (v2 >> 32); + putChar(buf, pos++, (char) (v2 >> 32)); } putLongLE(buf, pos, v1 & 0xffffffffffff0000L | (v2 >> 48)); return pos + 4; @@ -1084,9 +1076,9 @@ public static int writeInt64(final char[] buf, int pos, final long value) { putIntLE(buf, pos, (int) (v3 >> 16)); pos += 2; } else if (start == 1) { - buf[pos++] = (char) (v3 >> 32); + putChar(buf, pos++, (char) (v3 >> 32)); } - buf[pos] = (char) (v3 >> 48); + putChar(buf, pos, (char) (v3 >> 48)); putIntLE(buf, pos + 1, (int) (v2 >> 16)); putLongLE(buf, pos + 3, v1 & 0xffffffffffff0000L | (v2 >> 48)); return pos + 7; @@ -1101,10 +1093,10 @@ public static int writeInt64(final char[] buf, int pos, final long value) { putIntLE(buf, pos, (int) (v4 >> 16)); pos += 2; } else if (start == 1) { - buf[pos++] = (char) (v4 >> 32); + putChar(buf, pos++, (char) (v4 >> 32)); } - buf[pos] = (char) (v4 >> 48); - buf[pos + 1] = (char) (v3 >> 16); + putChar(buf, pos, (char) (v4 >> 48)); + putChar(buf, pos + 1, (char) (v3 >> 16)); putLongLE(buf, pos + 2, ((v2 & 0x0000ffffffff0000L) << 16) | (v3 >> 32)); putLongLE(buf, pos + 6, v1 & 0xffffffffffff0000L | (v2 >> 48)); return pos + 10; @@ -1119,9 +1111,9 @@ public static int writeInt64(final char[] buf, int pos, final long value) { putIntLE(buf, pos, (int) (v5 >> 16)); pos += 2; } else if (start == 1) { - buf[pos++] = (char) (v5 >> 32); + putChar(buf, pos++, (char) (v5 >> 32)); } - buf[pos] = (char) (v5 >> 48); + putChar(buf, pos, (char) (v5 >> 48)); putIntLE(buf, pos + 1, (int) (v4 >> 16)); putLongLE(buf, pos + 3, v3 & 0xffffffffffff0000L | (v4 >> 48)); putIntLE(buf, pos + 7, (int) (v2 >> 16)); @@ -1132,26 +1124,25 @@ public static int writeInt64(final char[] buf, int pos, final long value) { final int q6 = q5 / 1000; final long v5 = DIGITS_K_64[r5 & 0x3ff]; if (q6 == 0) { - int v = DIGITS_K_32[q5 & 0x3ff]; + long v = DIGITS_K_64[q5 & 0x3ff]; final int start = (byte) v; if (start == 0) { - buf[pos] = (char) (byte) (v >> 8); - buf[pos + 1] = (char) (byte) (v >> 16); + putIntUnaligned(buf, pos, (int) (v >> 16)); pos += 2; } else if (start == 1) { - buf[pos++] = (char) (byte) (v >> 16); + putChar(buf, pos++, (char) (v >> 32)); } - buf[pos++] = (char) (v >> 24); + putChar(buf, pos++, (char) (v >> 48)); } else { putLongLE(buf, pos, DIGITS_K_64[(q5 - q6 * 1000) & 0x3ff]); - buf[pos] = (char) (q6 + '0'); + putChar(buf, pos, (char) (q6 + '0')); pos += 4; } putIntLE(buf, pos, (int) (v5 >> 16)); putLongLE(buf, pos + 2, v4 & 0xffffffffffff0000L | (v5 >> 48)); - buf[pos + 6] = (char) (v3 >> 16); + putChar(buf, pos + 6, (char) (v3 >> 16)); putLongLE(buf, pos + 7, ((v2 & 0x0000ffffffff0000L) << 16) | (v3 >> 32)); putLongLE(buf, pos + 11, v1 & 0xffffffffffff0000L | (v2 >> 48)); return pos + 15; @@ -1161,7 +1152,7 @@ public static int writeInt8(final byte[] buf, int pos, final byte value) { int i; if (value < 0) { i = -value; - buf[pos++] = '-'; + putByte(buf, pos++, (byte) '-'); } else { i = value; } @@ -1172,17 +1163,17 @@ public static int writeInt8(final byte[] buf, int pos, final byte value) { putShortLE(buf, pos, (short) (v >> 8)); pos += 2; } else if (start == 1) { - buf[pos++] = (byte) (v >> 16); + putByte(buf, pos++, (byte) (v >> 16)); } - buf[pos] = (byte) (v >> 24); + putByte(buf, pos, (byte) (v >> 24)); return pos + 1; } - public static int writeInt8(final char[] buf, int pos, final byte value) { + public static int writeInt8(char[] buf, int pos, final byte value) { int i; if (value < 0) { i = -value; - buf[pos++] = '-'; + putChar(buf, pos++, '-'); } else { i = value; } @@ -1193,17 +1184,17 @@ public static int writeInt8(final char[] buf, int pos, final byte value) { putIntLE(buf, pos, (int) (v >> 16)); pos += 2; } else if (start == 1) { - buf[pos++] = (char) (v >> 32); + putChar(buf, pos++, (char) (v >> 32)); } - buf[pos] = (char) (v >> 48); + putChar(buf, pos, (char) (v >> 48)); return pos + 1; } - public static int writeInt16(final byte[] buf, int pos, final short value) { + public static int writeInt16(byte[] buf, int pos, final short value) { int i; if (value < 0) { i = -value; - buf[pos++] = '-'; + putByte(buf, pos++, (byte) '-'); } else { i = value; } @@ -1215,26 +1206,26 @@ public static int writeInt16(final byte[] buf, int pos, final short value) { putShortLE(buf, pos, (short) (v >> 8)); pos += 2; } else if (start == 1) { - buf[pos++] = (byte) (v >> 16); + putByte(buf, pos++, (byte) (v >> 16)); } - buf[pos] = (byte) (v >> 24); + putByte(buf, pos, (byte) (v >> 24)); return pos + 1; } final int q1 = i / 1000; final int v2 = DIGITS_K_32[q1 & 0x3ff]; if ((byte) v2 == 1) { - buf[pos++] = (byte) (v2 >> 16); + putByte(buf, pos++, (byte) (v2 >> 16)); } putIntLE(buf, pos, (DIGITS_K_32[(i - q1 * 1000) & 0x3ff]) & 0xffffff00 | (v2 >> 24)); return pos + 4; } - public static int writeInt16(final char[] buf, int pos, final short value) { + public static int writeInt16(char[] buf, int pos, final short value) { int i; if (value < 0) { i = -value; - buf[pos++] = '-'; + putChar(buf, pos++, '-'); } else { i = value; } @@ -1246,16 +1237,16 @@ public static int writeInt16(final char[] buf, int pos, final short value) { putIntLE(buf, pos, (int) (v >> 16)); pos += 2; } else if (start == 1) { - buf[pos++] = (char) (v >> 32); + putChar(buf, pos++, (char) (v >> 32)); } - buf[pos] = (char) (v >> 48); + putChar(buf, pos, (char) (v >> 48)); return pos + 1; } final int q1 = i / 1000; final long v2 = DIGITS_K_64[q1 & 0x3ff]; if ((byte) v2 == 1) { - buf[pos++] = (char) (v2 >> 32); + putChar(buf, pos++, (char) (v2 >> 32)); } putLongLE(buf, pos, DIGITS_K_64[(i - q1 * 1000) & 0x3ff] & 0xffffffffffff0000L | (v2 >> 48)); return pos + 4; @@ -1269,7 +1260,7 @@ public static int writeInt32(final byte[] buf, int pos, final int value) { return pos + MIN_INT_BYTES.length; } i = -value; - buf[pos++] = '-'; + putByte(buf, pos++, (byte) ('-')); } else { i = value; } @@ -1281,9 +1272,9 @@ public static int writeInt32(final byte[] buf, int pos, final int value) { putShortLE(buf, pos, (short) (v >> 8)); pos += 2; } else if (start == 1) { - buf[pos++] = (byte) (v >> 16); + putByte(buf, pos++, (byte) (v >> 16)); } - buf[pos] = (byte) (v >> 24); + putByte(buf, pos, (byte) (v >> 24)); return pos + 1; } @@ -1297,7 +1288,7 @@ public static int writeInt32(final byte[] buf, int pos, final int value) { putShortLE(buf, pos, (short) (v2 >> 8)); pos += 2; } else if (start == 1) { - buf[pos++] = (byte) (v2 >> 16); + putByte(buf, pos++, (byte) (v2 >> 16)); } putIntLE(buf, pos, v1 & 0xffffff00 | (v2 >> 24)); return pos + 4; @@ -1313,9 +1304,9 @@ public static int writeInt32(final byte[] buf, int pos, final int value) { putShortLE(buf, pos, (short) (v >> 8)); pos += 2; } else if (start == 1) { - buf[pos++] = (byte) (v >> 16); + putByte(buf, pos++, (byte) (v >> 16)); } - buf[pos++] = (byte) (v >> 24); + putByte(buf, pos++, (byte) (v >> 24)); } else { putIntLE(buf, pos, DIGITS_K_32[(q2 - q3 * 1000) & 0x3ff] & 0xffffff00 | (q3 + '0')); pos += 4; @@ -1334,7 +1325,7 @@ public static int writeInt32(final char[] buf, int pos, final int value) { return pos + MIN_INT_CHARS.length; } i = -value; - buf[pos++] = '-'; + putChar(buf, pos++, '-'); } else { i = value; } @@ -1345,9 +1336,9 @@ public static int writeInt32(final char[] buf, int pos, final int value) { putIntLE(buf, pos, (int) (v >> 16)); pos += 2; } else if (start == 1) { - buf[pos++] = (char) (v >> 32); + putChar(buf, pos++, (char) (v >> 32)); } - buf[pos] = (char) (v >> 48); + putChar(buf, pos, (char) (v >> 48)); return pos + 1; } final int q1 = i / 1000; @@ -1360,7 +1351,7 @@ public static int writeInt32(final char[] buf, int pos, final int value) { putIntLE(buf, pos, (int) (v2 >> 16)); pos += 2; } else if (start == 1) { - buf[pos++] = (char) (v2 >> 32); + putChar(buf, pos++, (char) (v2 >> 32)); } putLongLE(buf, pos, v1 & 0xffffffffffff0000L | (v2 >> 48)); return pos + 4; @@ -1376,12 +1367,12 @@ public static int writeInt32(final char[] buf, int pos, final int value) { putIntLE(buf, pos, (int) (v >> 16)); pos += 2; } else if (start == 1) { - buf[pos++] = (char) (v >> 32); + putChar(buf, pos++, (char) (v >> 32)); } - buf[pos++] = (char) (v >> 48); + putChar(buf, pos++, (char) (v >> 48)); } else { putLongLE(buf, pos, DIGITS_K_64[(q2 - q3 * 1000) & 0x3ff]); - buf[pos] = (char) (q3 + '0'); + putChar(buf, pos, (char) (q3 + '0')); pos += 4; } @@ -1390,6 +1381,10 @@ public static int writeInt32(final char[] buf, int pos, final int value) { return pos + 6; } + public static byte getByte(byte[] str, int pos) { + return UNSAFE.getByte(str, ARRAY_CHAR_BASE_OFFSET + pos); + } + public static char getChar(char[] buf, int pos) { return UNSAFE.getChar(buf, ARRAY_CHAR_BASE_OFFSET + ((long) pos << 1)); } @@ -1398,6 +1393,14 @@ public static char getChar(byte[] str, int pos) { return UNSAFE.getChar(str, ARRAY_CHAR_BASE_OFFSET + ((long) pos << 1)); } + public static void putByte(byte[] buf, int pos, byte v) { + UNSAFE.putByte(buf, ARRAY_CHAR_BASE_OFFSET + pos, v); + } + + public static void putChar(char[] buf, int pos, char v) { + UNSAFE.putChar(buf, ARRAY_CHAR_BASE_OFFSET + ((long) pos << 1), v); + } + public static void putShortLE(byte[] buf, int pos, short v) { UNSAFE.putShort(buf, ARRAY_BYTE_BASE_OFFSET + pos, convEndian(false, v)); } @@ -1442,13 +1445,28 @@ public static void putLongLE(byte[] buf, int pos, long v) { UNSAFE.putLong(buf, ARRAY_CHAR_BASE_OFFSET + pos, convEndian(false, v)); } - public static void putTrue(byte[] buf, int pos) { - UNSAFE.putInt(buf, ARRAY_CHAR_BASE_OFFSET + pos, TRUE); + public static int putBoolean(byte[] bytes, int off, boolean v) { + long address = ARRAY_CHAR_BASE_OFFSET + off; + if (v) { + UNSAFE.putInt(bytes, address, TRUE); + return off + 4; + } else { + UNSAFE.putByte(bytes, address, (byte) 'f'); + UNSAFE.putInt(bytes, address + 1, ALSE); + return off + 5; + } } - public static void putFalse(byte[] buf, int pos) { - UNSAFE.putByte(buf, ARRAY_CHAR_BASE_OFFSET + pos, (byte) 'f'); - UNSAFE.putInt(buf, ARRAY_CHAR_BASE_OFFSET + pos + 1, ALSE); + public static int putBoolean(char[] chars, int off, boolean v) { + long address = ARRAY_CHAR_BASE_OFFSET + ((long) off << 1); + if (v) { + UNSAFE.putLong(chars, address, TRUE_64); + return off + 4; + } else { + UNSAFE.putChar(chars, address, 'f'); + UNSAFE.putLong(chars, address + 2, ALSE_64); + return off + 5; + } } public static boolean isALSE(byte[] buf, int pos) { @@ -1459,15 +1477,6 @@ public static boolean isALSE(char[] buf, int pos) { return getLongUnaligned(buf, pos) == ALSE_64; } - public static void putTrue(char[] buf, int pos) { - UNSAFE.putLong(buf, ARRAY_CHAR_BASE_OFFSET + ((long) pos << 1), TRUE_64); - } - - public static void putFalse(char[] buf, int pos) { - UNSAFE.putChar(buf, ARRAY_CHAR_BASE_OFFSET + ((long) pos << 1), 'f'); - UNSAFE.putLong(buf, ARRAY_CHAR_BASE_OFFSET + ((pos + 1L) << 1), ALSE_64); - } - public static boolean isNULL(byte[] buf, int pos) { return getIntUnaligned(buf, pos) == NULL_32; } @@ -1485,10 +1494,11 @@ public static void putNULL(char[] buf, int pos) { } public static int digit4(char[] chars, int off) { - char c0 = getChar(chars, off), - c1 = getChar(chars, off + 1), - c2 = getChar(chars, off + 2), - c3 = getChar(chars, off + 3); + char c0, c1, c2, c3; + c0 = getChar(chars, off); + c1 = getChar(chars, off + 1); + c2 = getChar(chars, off + 2); + c3 = getChar(chars, off + 3); if ((c0 | c1 | c2 | c3) > 0x7f) { return -1; } @@ -1558,7 +1568,7 @@ public static int digit3(char[] chars, int off) { public static int digit3(byte[] bytes, int off) { return digit3( - getShortE(bytes, off) + getShortLE(bytes, off) | (UNSAFE.getByte(bytes, ARRAY_BYTE_BASE_OFFSET + off + 2) << 16) ); } @@ -1583,7 +1593,7 @@ public static int digit2(char[] chars, int off) { public static int digit2(byte[] bytes, int off) { return digit2( - getShortE(bytes, off) + getShortLE(bytes, off) ); } @@ -1645,11 +1655,20 @@ public static boolean isDigit(int ch) { return ch >= '0' && ch <= '9'; } - public static short getShortE(byte[] bytes, int offset) { + public static short getShortUnaligned(byte[] bytes, int offset) { + return UNSAFE.getShort(bytes, ARRAY_BYTE_BASE_OFFSET + offset); + } + + public static short getShortLE(byte[] bytes, int offset) { return convEndian(false, UNSAFE.getShort(bytes, ARRAY_BYTE_BASE_OFFSET + offset)); } + public static boolean isUTF8BOM(byte[] bytes, int off) { + // EF BB BF + return ((getIntLE(bytes, 0)) & 0xFFFFFF) == 0xBFBBEF; + } + public static int getIntBE(byte[] bytes, int offset) { return convEndian(true, UNSAFE.getInt(bytes, ARRAY_BYTE_BASE_OFFSET + offset)); @@ -1673,6 +1692,10 @@ public static long getLongBE(byte[] bytes, int offset) { UNSAFE.getLong(bytes, ARRAY_BYTE_BASE_OFFSET + offset)); } + public static long getLongUnaligned(byte[] bytes, int offset) { + return UNSAFE.getLong(bytes, ARRAY_BYTE_BASE_OFFSET + offset); + } + public static long getLongUnaligned(char[] bytes, int offset) { return UNSAFE.getLong(bytes, ARRAY_BYTE_BASE_OFFSET + ((long) offset << 1)); } diff --git a/core/src/test/java/com/alibaba/fastjson2/support/csv/CSVWriterTest.java b/core/src/test/java/com/alibaba/fastjson2/support/csv/CSVWriterTest.java index d051cc0485..842ce46a03 100644 --- a/core/src/test/java/com/alibaba/fastjson2/support/csv/CSVWriterTest.java +++ b/core/src/test/java/com/alibaba/fastjson2/support/csv/CSVWriterTest.java @@ -349,6 +349,26 @@ public void test7UTF8() throws Exception { } } + @Test + public void testBoolean() throws Exception { + try (CSVWriter csvWriter = CSVWriter.of(new StringWriter())) { + csvWriter.writeBoolean(true); + csvWriter.writeComma(); + csvWriter.writeBoolean(false); + assertEquals("true,false", csvWriter.toString()); + } + } + + @Test + public void testBooleanUTF8() throws Exception { + try (CSVWriter csvWriter = CSVWriter.of(new ByteArrayOutputStream())) { + csvWriter.writeBoolean(true); + csvWriter.writeComma(); + csvWriter.writeBoolean(false); + assertEquals("true,false", csvWriter.toString()); + } + } + @Test public void testWriteInstant() throws Exception { LocalDate date = LocalDate.of(2018, 7, 12);