Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

JSON configuration and Jackson Streaming Object Processor #5225

Merged
merged 56 commits into from
Jun 18, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
56 commits
Select commit Hold shift + click to select a range
1320749
JSON configuration and Processor
devinrsmith Apr 11, 2024
bb1d02e
spotless
devinrsmith Apr 11, 2024
5c7dde9
review response
devinrsmith Apr 12, 2024
44f4bc9
spotless
devinrsmith Apr 15, 2024
c567c21
Boxed builders
devinrsmith Apr 15, 2024
60fad28
Make universe implementation / documentation detail
devinrsmith Apr 15, 2024
2358a24
f
devinrsmith Apr 16, 2024
50ec4cb
Rename to Value
devinrsmith Apr 16, 2024
5dc5aa8
unmodifiable set
devinrsmith Apr 16, 2024
26ff817
f
devinrsmith Apr 16, 2024
14eadba
f
devinrsmith Apr 16, 2024
5c65852
f
devinrsmith Apr 17, 2024
979bdfa
ValueInnerRepeaterProcessor
devinrsmith Apr 17, 2024
fd3f6ed
ValueProcessor knows types
devinrsmith Apr 17, 2024
a16b30f
f
devinrsmith Apr 18, 2024
f564bc4
f
devinrsmith May 13, 2024
f4c674e
f
devinrsmith May 21, 2024
6f45aa7
some review responses
devinrsmith May 22, 2024
6d4ebef
f
devinrsmith May 23, 2024
32841a5
f
devinrsmith May 24, 2024
7a1a2d3
Merge remote-tracking branch 'upstream/main' into json-config-review
devinrsmith May 24, 2024
2d73895
exceptions
devinrsmith May 24, 2024
c6f098b
tests
devinrsmith May 24, 2024
cf81769
Rename tests
devinrsmith May 28, 2024
8d1e377
f
devinrsmith May 28, 2024
5828f1a
stuff
devinrsmith May 28, 2024
5022c2d
Tests and stuff
devinrsmith May 28, 2024
88db86d
Typed object improvements
devinrsmith May 28, 2024
f4b60c2
Array maths
devinrsmith May 29, 2024
52e5932
deeply nested array test
devinrsmith May 29, 2024
dbb66de
array stuff
devinrsmith May 29, 2024
0c0760e
kv mixin
devinrsmith May 29, 2024
c9c20e0
mixin consolidation
devinrsmith May 29, 2024
f979b56
discriminated object
devinrsmith May 29, 2024
2338fa0
outputSize
devinrsmith May 29, 2024
34dda0c
cleanup
devinrsmith May 29, 2024
9fccd9d
Instant array types
devinrsmith May 29, 2024
c4cae88
Merge remote-tracking branch 'upstream/main' into json-config-review
devinrsmith May 29, 2024
a389e88
Introduce ObjectChunkDeepEquals
devinrsmith May 30, 2024
91c2990
Move LongRepeaterImpl to LongMixin
devinrsmith May 30, 2024
1526eb4
Remove some exceptions
devinrsmith May 30, 2024
30ef952
math util
devinrsmith May 30, 2024
c0ab02a
Update python
devinrsmith May 30, 2024
08e44de
Review responses
devinrsmith May 30, 2024
aeb4628
Merge remote-tracking branch 'upstream/main' into json-config-review
devinrsmith Jun 11, 2024
6a6f383
Add python json tests
devinrsmith Jun 12, 2024
c7fc14b
Fix test
devinrsmith Jun 12, 2024
10b7794
Undo ChunkEquals interface changes in light of #5605
devinrsmith Jun 12, 2024
c26304e
Fix on_null, on_missing, add tests
devinrsmith Jun 13, 2024
fb9d759
object_processor_spec, jackson provider python construction test
devinrsmith Jun 13, 2024
258a8b8
Add JacksonProvider documentation
devinrsmith Jun 13, 2024
1683d9e
Review points
devinrsmith Jun 13, 2024
c53262c
Python _val
devinrsmith Jun 13, 2024
3cf7e9f
rename ObjectKvValue to ObjectEntriesValue
devinrsmith Jun 13, 2024
7e81e41
Review responses
devinrsmith Jun 14, 2024
b319512
Add high-level example documentation is python json module
devinrsmith Jun 17, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 37 additions & 0 deletions Base/src/main/java/io/deephaven/base/MathUtil.java
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,11 @@
*/
public class MathUtil {

/**
* The maximum power of 2.
*/
public static final int MAX_POWER_OF_2 = 1 << 30;

/**
* Compute ceil(log2(x)). See {@link Integer#numberOfLeadingZeros(int)}.
*
Expand Down Expand Up @@ -108,4 +113,36 @@ public static int base10digits(int n) {
}
return base10guess;
}

/**
* Rounds up to the next power of 2 for {@code x}; if {@code x} is already a power of 2, {@code x} will be returned.
* Values outside the range {@code 1 <= x <= MAX_POWER_OF_2} will return {@code 1}.
*
* <p>
* Equivalent to {@code Math.max(Integer.highestOneBit(x - 1) << 1, 1)}.
*
* @param x the value
* @return the next power of 2 for {@code x}
* @see #MAX_POWER_OF_2
*/
public static int roundUpPowerOf2(int x) {
return Math.max(Integer.highestOneBit(x - 1) << 1, 1);
}

/**
* Rounds up to the next power of 2 for {@code size <= MAX_POWER_OF_2}, otherwise returns
* {@link ArrayUtil#MAX_ARRAY_SIZE}.
*
* <p>
* Equivalent to {@code size <= MAX_POWER_OF_2 ? roundUpPowerOf2(size) : ArrayUtil.MAX_ARRAY_SIZE}.
*
* @param size the size
* @return the
* @see #MAX_POWER_OF_2
* @see #roundUpPowerOf2(int)
* @see ArrayUtil#MAX_ARRAY_SIZE
*/
public static int roundUpArraySize(int size) {
return size <= MAX_POWER_OF_2 ? roundUpPowerOf2(size) : ArrayUtil.MAX_ARRAY_SIZE;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
// @formatter:off
package io.deephaven.base.ringbuffer;

import io.deephaven.base.ArrayUtil;
import io.deephaven.base.MathUtil;
import io.deephaven.base.verify.Assert;

import java.io.Serializable;
Expand All @@ -20,8 +20,6 @@
* determination of storage indices through a mask operation.
*/
public class ByteRingBuffer implements Serializable {
/** Maximum capacity is the highest power of two that can be allocated (i.e. <= than ArrayUtil.MAX_ARRAY_SIZE). */
static final int RING_BUFFER_MAX_CAPACITY = Integer.highestOneBit(ArrayUtil.MAX_ARRAY_SIZE);
static final long FIXUP_THRESHOLD = 1L << 62;
final boolean growable;
byte[] storage;
Expand All @@ -45,21 +43,13 @@ public ByteRingBuffer(int capacity) {
* @param growable whether to allow growth when the buffer is full.
*/
public ByteRingBuffer(int capacity, boolean growable) {
Assert.leq(capacity, "ByteRingBuffer capacity", RING_BUFFER_MAX_CAPACITY);
Assert.leq(capacity, "ByteRingBuffer capacity", MathUtil.MAX_POWER_OF_2);

this.growable = growable;

// use next larger power of 2 for our storage
final int newCapacity;
if (capacity < 2) {
// sensibly handle the size=0 and size=1 cases
newCapacity = 1;
} else {
newCapacity = Integer.highestOneBit(capacity - 1) << 1;
}

// reset the data structure members
storage = new byte[newCapacity];
storage = new byte[MathUtil.roundUpPowerOf2(capacity)];
mask = storage.length - 1;
tail = head = 0;
}
Expand All @@ -73,9 +63,9 @@ protected void grow(int increase) {
final int size = size();
final long newCapacity = (long) storage.length + increase;
// assert that we are not asking for the impossible
Assert.leq(newCapacity, "ByteRingBuffer capacity", RING_BUFFER_MAX_CAPACITY);
Assert.leq(newCapacity, "ByteRingBuffer capacity", MathUtil.MAX_POWER_OF_2);

final byte[] newStorage = new byte[Integer.highestOneBit((int) newCapacity - 1) << 1];
final byte[] newStorage = new byte[MathUtil.roundUpPowerOf2((int) newCapacity)];

// move the current data to the new buffer
copyRingBufferToArray(newStorage);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
//
package io.deephaven.base.ringbuffer;

import io.deephaven.base.ArrayUtil;
import io.deephaven.base.MathUtil;
import io.deephaven.base.verify.Assert;

import java.io.Serializable;
Expand All @@ -16,8 +16,6 @@
* determination of storage indices through a mask operation.
*/
public class CharRingBuffer implements Serializable {
/** Maximum capacity is the highest power of two that can be allocated (i.e. <= than ArrayUtil.MAX_ARRAY_SIZE). */
static final int RING_BUFFER_MAX_CAPACITY = Integer.highestOneBit(ArrayUtil.MAX_ARRAY_SIZE);
static final long FIXUP_THRESHOLD = 1L << 62;
final boolean growable;
char[] storage;
Expand All @@ -41,21 +39,13 @@ public CharRingBuffer(int capacity) {
* @param growable whether to allow growth when the buffer is full.
*/
public CharRingBuffer(int capacity, boolean growable) {
Assert.leq(capacity, "CharRingBuffer capacity", RING_BUFFER_MAX_CAPACITY);
Assert.leq(capacity, "CharRingBuffer capacity", MathUtil.MAX_POWER_OF_2);

this.growable = growable;

// use next larger power of 2 for our storage
final int newCapacity;
if (capacity < 2) {
// sensibly handle the size=0 and size=1 cases
newCapacity = 1;
} else {
newCapacity = Integer.highestOneBit(capacity - 1) << 1;
}

// reset the data structure members
storage = new char[newCapacity];
storage = new char[MathUtil.roundUpPowerOf2(capacity)];
mask = storage.length - 1;
tail = head = 0;
}
Expand All @@ -69,9 +59,9 @@ protected void grow(int increase) {
final int size = size();
final long newCapacity = (long) storage.length + increase;
// assert that we are not asking for the impossible
Assert.leq(newCapacity, "CharRingBuffer capacity", RING_BUFFER_MAX_CAPACITY);
Assert.leq(newCapacity, "CharRingBuffer capacity", MathUtil.MAX_POWER_OF_2);

final char[] newStorage = new char[Integer.highestOneBit((int) newCapacity - 1) << 1];
final char[] newStorage = new char[MathUtil.roundUpPowerOf2((int) newCapacity)];

// move the current data to the new buffer
copyRingBufferToArray(newStorage);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
// @formatter:off
package io.deephaven.base.ringbuffer;

import io.deephaven.base.ArrayUtil;
import io.deephaven.base.MathUtil;
import io.deephaven.base.verify.Assert;

import java.io.Serializable;
Expand All @@ -20,8 +20,6 @@
* determination of storage indices through a mask operation.
*/
public class DoubleRingBuffer implements Serializable {
/** Maximum capacity is the highest power of two that can be allocated (i.e. <= than ArrayUtil.MAX_ARRAY_SIZE). */
static final int RING_BUFFER_MAX_CAPACITY = Integer.highestOneBit(ArrayUtil.MAX_ARRAY_SIZE);
static final long FIXUP_THRESHOLD = 1L << 62;
final boolean growable;
double[] storage;
Expand All @@ -45,21 +43,13 @@ public DoubleRingBuffer(int capacity) {
* @param growable whether to allow growth when the buffer is full.
*/
public DoubleRingBuffer(int capacity, boolean growable) {
Assert.leq(capacity, "DoubleRingBuffer capacity", RING_BUFFER_MAX_CAPACITY);
Assert.leq(capacity, "DoubleRingBuffer capacity", MathUtil.MAX_POWER_OF_2);

this.growable = growable;

// use next larger power of 2 for our storage
final int newCapacity;
if (capacity < 2) {
// sensibly handle the size=0 and size=1 cases
newCapacity = 1;
} else {
newCapacity = Integer.highestOneBit(capacity - 1) << 1;
}

// reset the data structure members
storage = new double[newCapacity];
storage = new double[MathUtil.roundUpPowerOf2(capacity)];
mask = storage.length - 1;
tail = head = 0;
}
Expand All @@ -73,9 +63,9 @@ protected void grow(int increase) {
final int size = size();
final long newCapacity = (long) storage.length + increase;
// assert that we are not asking for the impossible
Assert.leq(newCapacity, "DoubleRingBuffer capacity", RING_BUFFER_MAX_CAPACITY);
Assert.leq(newCapacity, "DoubleRingBuffer capacity", MathUtil.MAX_POWER_OF_2);

final double[] newStorage = new double[Integer.highestOneBit((int) newCapacity - 1) << 1];
final double[] newStorage = new double[MathUtil.roundUpPowerOf2((int) newCapacity)];

// move the current data to the new buffer
copyRingBufferToArray(newStorage);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
// @formatter:off
package io.deephaven.base.ringbuffer;

import io.deephaven.base.ArrayUtil;
import io.deephaven.base.MathUtil;
import io.deephaven.base.verify.Assert;

import java.io.Serializable;
Expand All @@ -20,8 +20,6 @@
* determination of storage indices through a mask operation.
*/
public class FloatRingBuffer implements Serializable {
/** Maximum capacity is the highest power of two that can be allocated (i.e. <= than ArrayUtil.MAX_ARRAY_SIZE). */
static final int RING_BUFFER_MAX_CAPACITY = Integer.highestOneBit(ArrayUtil.MAX_ARRAY_SIZE);
static final long FIXUP_THRESHOLD = 1L << 62;
final boolean growable;
float[] storage;
Expand All @@ -45,21 +43,13 @@ public FloatRingBuffer(int capacity) {
* @param growable whether to allow growth when the buffer is full.
*/
public FloatRingBuffer(int capacity, boolean growable) {
Assert.leq(capacity, "FloatRingBuffer capacity", RING_BUFFER_MAX_CAPACITY);
Assert.leq(capacity, "FloatRingBuffer capacity", MathUtil.MAX_POWER_OF_2);

this.growable = growable;

// use next larger power of 2 for our storage
final int newCapacity;
if (capacity < 2) {
// sensibly handle the size=0 and size=1 cases
newCapacity = 1;
} else {
newCapacity = Integer.highestOneBit(capacity - 1) << 1;
}

// reset the data structure members
storage = new float[newCapacity];
storage = new float[MathUtil.roundUpPowerOf2(capacity)];
mask = storage.length - 1;
tail = head = 0;
}
Expand All @@ -73,9 +63,9 @@ protected void grow(int increase) {
final int size = size();
final long newCapacity = (long) storage.length + increase;
// assert that we are not asking for the impossible
Assert.leq(newCapacity, "FloatRingBuffer capacity", RING_BUFFER_MAX_CAPACITY);
Assert.leq(newCapacity, "FloatRingBuffer capacity", MathUtil.MAX_POWER_OF_2);

final float[] newStorage = new float[Integer.highestOneBit((int) newCapacity - 1) << 1];
final float[] newStorage = new float[MathUtil.roundUpPowerOf2((int) newCapacity)];

// move the current data to the new buffer
copyRingBufferToArray(newStorage);
Expand Down
20 changes: 5 additions & 15 deletions Base/src/main/java/io/deephaven/base/ringbuffer/IntRingBuffer.java
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
// @formatter:off
package io.deephaven.base.ringbuffer;

import io.deephaven.base.ArrayUtil;
import io.deephaven.base.MathUtil;
import io.deephaven.base.verify.Assert;

import java.io.Serializable;
Expand All @@ -20,8 +20,6 @@
* determination of storage indices through a mask operation.
*/
public class IntRingBuffer implements Serializable {
/** Maximum capacity is the highest power of two that can be allocated (i.e. <= than ArrayUtil.MAX_ARRAY_SIZE). */
static final int RING_BUFFER_MAX_CAPACITY = Integer.highestOneBit(ArrayUtil.MAX_ARRAY_SIZE);
static final long FIXUP_THRESHOLD = 1L << 62;
final boolean growable;
int[] storage;
Expand All @@ -45,21 +43,13 @@ public IntRingBuffer(int capacity) {
* @param growable whether to allow growth when the buffer is full.
*/
public IntRingBuffer(int capacity, boolean growable) {
Assert.leq(capacity, "IntRingBuffer capacity", RING_BUFFER_MAX_CAPACITY);
Assert.leq(capacity, "IntRingBuffer capacity", MathUtil.MAX_POWER_OF_2);

this.growable = growable;

// use next larger power of 2 for our storage
final int newCapacity;
if (capacity < 2) {
// sensibly handle the size=0 and size=1 cases
newCapacity = 1;
} else {
newCapacity = Integer.highestOneBit(capacity - 1) << 1;
}

// reset the data structure members
storage = new int[newCapacity];
storage = new int[MathUtil.roundUpPowerOf2(capacity)];
mask = storage.length - 1;
tail = head = 0;
}
Expand All @@ -73,9 +63,9 @@ protected void grow(int increase) {
final int size = size();
final long newCapacity = (long) storage.length + increase;
// assert that we are not asking for the impossible
Assert.leq(newCapacity, "IntRingBuffer capacity", RING_BUFFER_MAX_CAPACITY);
Assert.leq(newCapacity, "IntRingBuffer capacity", MathUtil.MAX_POWER_OF_2);

final int[] newStorage = new int[Integer.highestOneBit((int) newCapacity - 1) << 1];
final int[] newStorage = new int[MathUtil.roundUpPowerOf2((int) newCapacity)];

// move the current data to the new buffer
copyRingBufferToArray(newStorage);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
// @formatter:off
package io.deephaven.base.ringbuffer;

import io.deephaven.base.ArrayUtil;
import io.deephaven.base.MathUtil;
import io.deephaven.base.verify.Assert;

import java.io.Serializable;
Expand All @@ -20,8 +20,6 @@
* determination of storage indices through a mask operation.
*/
public class LongRingBuffer implements Serializable {
/** Maximum capacity is the highest power of two that can be allocated (i.e. <= than ArrayUtil.MAX_ARRAY_SIZE). */
static final int RING_BUFFER_MAX_CAPACITY = Integer.highestOneBit(ArrayUtil.MAX_ARRAY_SIZE);
static final long FIXUP_THRESHOLD = 1L << 62;
final boolean growable;
long[] storage;
Expand All @@ -45,21 +43,13 @@ public LongRingBuffer(int capacity) {
* @param growable whether to allow growth when the buffer is full.
*/
public LongRingBuffer(int capacity, boolean growable) {
Assert.leq(capacity, "LongRingBuffer capacity", RING_BUFFER_MAX_CAPACITY);
Assert.leq(capacity, "LongRingBuffer capacity", MathUtil.MAX_POWER_OF_2);

this.growable = growable;

// use next larger power of 2 for our storage
final int newCapacity;
if (capacity < 2) {
// sensibly handle the size=0 and size=1 cases
newCapacity = 1;
} else {
newCapacity = Integer.highestOneBit(capacity - 1) << 1;
}

// reset the data structure members
storage = new long[newCapacity];
storage = new long[MathUtil.roundUpPowerOf2(capacity)];
mask = storage.length - 1;
tail = head = 0;
}
Expand All @@ -73,9 +63,9 @@ protected void grow(int increase) {
final int size = size();
final long newCapacity = (long) storage.length + increase;
// assert that we are not asking for the impossible
Assert.leq(newCapacity, "LongRingBuffer capacity", RING_BUFFER_MAX_CAPACITY);
Assert.leq(newCapacity, "LongRingBuffer capacity", MathUtil.MAX_POWER_OF_2);

final long[] newStorage = new long[Integer.highestOneBit((int) newCapacity - 1) << 1];
final long[] newStorage = new long[MathUtil.roundUpPowerOf2((int) newCapacity)];

// move the current data to the new buffer
copyRingBufferToArray(newStorage);
Expand Down
Loading
Loading