Skip to content

Commit

Permalink
refactor
Browse files Browse the repository at this point in the history
  • Loading branch information
@wenshao
wenshao committed Jan 15, 2025
1 parent 45d473c commit c8ba303
Show file tree
Hide file tree
Showing 8 changed files with 87 additions and 93 deletions.
8 changes: 4 additions & 4 deletions benchmark/src/test/java/com/alibaba/fastjson2/benchmark/CSVBankTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@

public class CSVBankTest {
static final CSVBank benchmark = new CSVBank();
static final int LOOP = 1000;
static final int LOOP = 10000;

public static void fastjson2() {
for (int j = 0; j < 5; j++) {
Expand All @@ -16,7 +16,7 @@ public static void fastjson2() {
System.out.println("fastjson2 millis : " + millis);
// zulu8.68.0.21 : 213
// zulu11.62.17 : 148
// zulu17.40.19 : 150
// zulu17.40.19 : 150 1746 1639
}
}

Expand Down Expand Up @@ -49,8 +49,8 @@ public static void cainiao() throws Exception {
}

public static void main(String[] args) throws Exception {
// fastjson2();
univocity();
fastjson2();
// univocity();
// cainiao();
}
}
8 changes: 1 addition & 7 deletions core/src/main/java/com/alibaba/fastjson2/JSONWriterUTF16JDK9UF.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,13 +78,7 @@ public void writeBool(boolean value) {
if ((context.features & WriteBooleanAsNumber.mask) != 0) {
chars[off++] = value ? '1' : '0';
} else {
if (value) {
IOUtils.putTrue(chars, off);
off += 4;
} else {
IOUtils.putFalse(chars, off);
off += 5;
}
off = IOUtils.putBoolean(chars, off, value);
}
this.off = off;
}
Expand Down
8 changes: 1 addition & 7 deletions core/src/main/java/com/alibaba/fastjson2/JSONWriterUTF8.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2927,13 +2927,7 @@ public void writeBool(boolean value) {
if ((context.features & WriteBooleanAsNumber.mask) != 0) {
bytes[off++] = (byte) (value ? '1' : '0');
} else {
if (value) {
IOUtils.putTrue(bytes, off);
off += 4;
} else {
IOUtils.putFalse(bytes, off);
off += 5;
}
off = IOUtils.putBoolean(bytes, off, value);
}
this.off = off;
}
Expand Down
20 changes: 7 additions & 13 deletions core/src/main/java/com/alibaba/fastjson2/support/csv/CSVReaderUTF16.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import java.util.function.Consumer;
import java.util.function.Function;

import static com.alibaba.fastjson2.support.csv.CSVReaderUTF8.containsQuoteOrLineSeparator;
import static com.alibaba.fastjson2.util.DateUtils.DEFAULT_ZONE_ID;

final class CSVReaderUTF16<T>
Expand Down Expand Up @@ -97,19 +98,11 @@ protected boolean seekLine() throws IOException {
for (int k = 0; k < 3; ++k) {
lineTerminated = false;

for (int i = off; i < end; i++) {
if (i + 4 < end) {
char b0 = buf[i];
char b1 = buf[i + 1];
char b2 = buf[i + 2];
char b3 = buf[i + 3];
if (b0 > '"' && b1 > '"' && b2 > '"' && b3 > '"') {
lineSize += 4;
i += 3;
continue;
}
}

int i = off, end = this.end;
while (i + 4 < end && !containsQuoteOrLineSeparator(IOUtils.getLongUnaligned(buf, i))) {
i += 4;
}
for (; i < end; i++) {
char ch = buf[i];
if (ch == '"') {
lineSize++;
Expand Down Expand Up @@ -194,6 +187,7 @@ protected boolean seekLine() throws IOException {
}
} else {
end += cnt;
this.end = end;
continue;
}
}
Expand Down
36 changes: 29 additions & 7 deletions core/src/main/java/com/alibaba/fastjson2/support/csv/CSVReaderUTF8.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,26 @@ final class CSVReaderUTF8<T>
this.valueConsumer = valueConsumer;
}

static boolean containsQuoteOrLineSeparator(long v) {
/*
for (int i = 0; i < 8; ++i) {
byte c = (byte) v;
if (c == '"' || c == '\n' || c == '\r') {
return true;
}
v >>>= 8;
}
return false;
*/
long x22 = v ^ 0x2222222222222222L; // " -> 0x22
long x0a = v ^ 0x0A0A0A0A0A0A0A0AL; // \n -> 0x0a
long x0d = v ^ 0x0D0D0D0D0D0D0D0DL; // \r -> 0x0d
x22 = (x22 - 0x0101010101010101L) & ~x22;
x0a = (x0a - 0x0101010101010101L) & ~x0a;
x0d = (x0d - 0x0101010101010101L) & ~x0d;
return ((x22 | x0a | x0d) & 0x8080808080808080L) != 0;
}

protected boolean seekLine() throws IOException {
byte[] buf = this.buf;
int off = this.off;
Expand All @@ -100,20 +120,21 @@ protected boolean seekLine() throws IOException {
}
this.end = cnt;

if (end > 3) {
// UTF8-BOM EF BB BF
if (buf[0] == -17 && buf[1] == -69 && buf[2] == -65) {
off = 3;
lineNextStart = off;
}
if (end > 4 && IOUtils.isUTF8BOM(buf, 0)) {
off = 3;
lineNextStart = off;
}
}
}

for (int k = 0; k < 3; ++k) {
lineTerminated = false;

for (int i = off; i < end; i++) {
int i = off, end = this.end;
while (i + 8 < end && !containsQuoteOrLineSeparator(IOUtils.getLongUnaligned(buf, i))) {
i += 8;
}
for (; i < end; i++) {
byte ch = buf[i];
if (ch == '"') {
lineSize++;
Expand Down Expand Up @@ -198,6 +219,7 @@ protected boolean seekLine() throws IOException {
}
} else {
end += cnt;
this.end = end;
continue;
}
}
Expand Down
22 changes: 4 additions & 18 deletions core/src/main/java/com/alibaba/fastjson2/support/csv/CSVWriterUTF16.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,6 @@
import java.time.LocalDateTime;
import java.time.ZoneId;

import static com.alibaba.fastjson2.util.IOUtils.*;

final class CSVWriterUTF16
extends CSVWriter {
final Writer out;
Expand Down Expand Up @@ -52,17 +50,9 @@ public void writeLine() {
chars[off++] = '\n';
}

public void writeBoolean(boolean booleanValue) {
int size = booleanValue ? 4 : 5;
checkCapacity(size);
char[] chars = this.chars;
int off = this.off;
if (booleanValue) {
IOUtils.putTrue(chars, off);
} else {
IOUtils.putFalse(chars, off);
}
this.off = off + size;
public void writeBoolean(boolean v) {
checkCapacity(5);
this.off = IOUtils.putBoolean(chars, off, v);
}

public void writeInt64(long longValue) {
Expand All @@ -88,11 +78,7 @@ public void writeDateTime19(
int off = this.off;
off = IOUtils.writeLocalDate(chars, off, year, month, dayOfMonth);
chars[off] = ' ';
writeDigitPair(chars, off + 1, hour);
chars[off + 3] = ':';
writeDigitPair(chars, off + 4, minute);
chars[off + 6] = ':';
writeDigitPair(chars, off + 7, second);
IOUtils.writeLocalTime(chars, off + 1, hour, minute, second);
this.off = off + 9;
}

Expand Down
24 changes: 4 additions & 20 deletions core/src/main/java/com/alibaba/fastjson2/support/csv/CSVWriterUTF8.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,6 @@
import java.time.LocalDateTime;
import java.time.ZoneId;

import static com.alibaba.fastjson2.util.IOUtils.*;

final class CSVWriterUTF8
extends CSVWriter {
final OutputStream out;
Expand Down Expand Up @@ -55,28 +53,18 @@ public void writeLine() {
bytes[off++] = '\n';
}

public void writeBoolean(boolean booleanValue) {
int size = booleanValue ? 4 : 5;
checkCapacity(size);
byte[] chars = this.bytes;
int off = this.off;
if (booleanValue) {
IOUtils.putTrue(chars, off);
} else {
IOUtils.putFalse(chars, off);
}
this.off = off + size;
public void writeBoolean(boolean v) {
checkCapacity(5);
this.off = IOUtils.putBoolean(this.bytes, off, v);
}

public void writeInt64(long longValue) {
checkCapacity(20); // -9223372036854775808

off = IOUtils.writeInt64(bytes, off, longValue);
}

public void writeDateYYYMMDD10(int year, int month, int dayOfMonth) {
checkCapacity(10);

off = IOUtils.writeLocalDate(bytes, off, year, month, dayOfMonth);
}

Expand All @@ -94,11 +82,7 @@ public void writeDateTime19(
int off = this.off;
off = IOUtils.writeLocalDate(bytes, off, year, month, dayOfMonth);
bytes[off] = ' ';
writeDigitPair(bytes, off + 1, hour);
bytes[off + 3] = ':';
writeDigitPair(bytes, off + 4, minute);
bytes[off + 6] = ':';
writeDigitPair(bytes, off + 7, second);
IOUtils.writeLocalTime(bytes, off + 1, hour, minute, second);
this.off = off + 9;
}

Expand Down
54 changes: 37 additions & 17 deletions core/src/main/java/com/alibaba/fastjson2/util/IOUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1381,6 +1381,11 @@ public static int writeInt32(final char[] buf, int pos, final int value) {
return pos + 6;
}

public static byte getByte(byte[] str, int pos) {
return UNSAFE.getByte(str, ARRAY_CHAR_BASE_OFFSET + pos);
}


public static char getChar(char[] buf, int pos) {
return UNSAFE.getChar(buf, ARRAY_CHAR_BASE_OFFSET + ((long) pos << 1));
}
Expand Down Expand Up @@ -1441,13 +1446,28 @@ public static void putLongLE(byte[] buf, int pos, long v) {
UNSAFE.putLong(buf, ARRAY_CHAR_BASE_OFFSET + pos, convEndian(false, v));
}

public static void putTrue(byte[] buf, int pos) {
UNSAFE.putInt(buf, ARRAY_CHAR_BASE_OFFSET + pos, TRUE);
public static int putBoolean(byte[] bytes, int off, boolean v) {
long address = ARRAY_CHAR_BASE_OFFSET + off;
if (v) {
UNSAFE.putInt(bytes, address, TRUE);
return off + 4;
} else {
UNSAFE.putByte(bytes, address, (byte) 'f');
UNSAFE.putInt(bytes, address + 1, ALSE);
return off + 5;
}
}

public static void putFalse(byte[] buf, int pos) {
UNSAFE.putByte(buf, ARRAY_CHAR_BASE_OFFSET + pos, (byte) 'f');
UNSAFE.putInt(buf, ARRAY_CHAR_BASE_OFFSET + pos + 1, ALSE);
public static int putBoolean(char[] chars, int off, boolean v) {
long address = ARRAY_CHAR_BASE_OFFSET + ((long) off << 1);
if (v) {
UNSAFE.putLong(chars, address, TRUE_64);
return off + 4;
} else {
UNSAFE.putChar(chars, address, 'f');
UNSAFE.putLong(chars, address + 2, ALSE_64);
return off + 5;
}
}

public static boolean isALSE(byte[] buf, int pos) {
Expand All @@ -1458,15 +1478,6 @@ public static boolean isALSE(char[] buf, int pos) {
return getLongUnaligned(buf, pos) == ALSE_64;
}

public static void putTrue(char[] buf, int pos) {
UNSAFE.putLong(buf, ARRAY_CHAR_BASE_OFFSET + ((long) pos << 1), TRUE_64);
}

public static void putFalse(char[] buf, int pos) {
UNSAFE.putChar(buf, ARRAY_CHAR_BASE_OFFSET + ((long) pos << 1), 'f');
UNSAFE.putLong(buf, ARRAY_CHAR_BASE_OFFSET + ((pos + 1L) << 1), ALSE_64);
}

public static boolean isNULL(byte[] buf, int pos) {
return getIntUnaligned(buf, pos) == NULL_32;
}
Expand Down Expand Up @@ -1558,7 +1569,7 @@ public static int digit3(char[] chars, int off) {

public static int digit3(byte[] bytes, int off) {
return digit3(
getShortE(bytes, off)
getShortLE(bytes, off)
| (UNSAFE.getByte(bytes, ARRAY_BYTE_BASE_OFFSET + off + 2) << 16)
);
}
Expand All @@ -1583,7 +1594,7 @@ public static int digit2(char[] chars, int off) {

public static int digit2(byte[] bytes, int off) {
return digit2(
getShortE(bytes, off)
getShortLE(bytes, off)
);
}

Expand Down Expand Up @@ -1645,11 +1656,16 @@ public static boolean isDigit(int ch) {
return ch >= '0' && ch <= '9';
}

public static short getShortE(byte[] bytes, int offset) {
public static short getShortLE(byte[] bytes, int offset) {
return convEndian(false,
UNSAFE.getShort(bytes, ARRAY_BYTE_BASE_OFFSET + offset));
}

public static boolean isUTF8BOM(byte[] bytes, int off) {
// EF BB BF
return ((getIntLE(bytes, 0)) & 0xFFFFFF) == 0xBFBBEF;
}

public static int getIntBE(byte[] bytes, int offset) {
return convEndian(true,
UNSAFE.getInt(bytes, ARRAY_BYTE_BASE_OFFSET + offset));
Expand All @@ -1673,6 +1689,10 @@ public static long getLongBE(byte[] bytes, int offset) {
UNSAFE.getLong(bytes, ARRAY_BYTE_BASE_OFFSET + offset));
}

public static long getLongUnaligned(byte[] bytes, int offset) {
return UNSAFE.getLong(bytes, ARRAY_BYTE_BASE_OFFSET + offset);
}

public static long getLongUnaligned(char[] bytes, int offset) {
return UNSAFE.getLong(bytes, ARRAY_BYTE_BASE_OFFSET + ((long) offset << 1));
}
Expand Down

0 comments on commit c8ba303

Please sign in to comment.